


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__snake_case            =   logging.get_logger(__name__)

__snake_case            =   {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__snake_case            =   {
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

__snake_case            =   {
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}




class    UpperCAmelCase					(       lowercase__    ):
   lowercase						= VOCAB_FILES_NAMES
   lowercase						= PRETRAINED_VOCAB_FILES_MAP
   lowercase						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowercase						= ["""input_ids""", """attention_mask"""]
   lowercase						= RobertaTokenizer




   def __init__(						self							:		List[str]       ,       __magic_name__							:		Optional[Any]=None       ,       __magic_name__							:		List[str]=None       ,       __magic_name__							:		Any=None       ,       __magic_name__							:		Optional[int]="replace"       ,       __magic_name__							:		List[Any]="<s>"       ,       __magic_name__							:		str="</s>"       ,       __magic_name__							:		List[Any]="</s>"       ,       __magic_name__							:		int="<s>"       ,       __magic_name__							:		Dict="<unk>"       ,       __magic_name__							:		List[Any]="<pad>"       ,       __magic_name__							:		List[str]="<mask>"       ,       __magic_name__							:		Tuple=False       ,       __magic_name__							:		str=True       ,       **__magic_name__							:		Tuple       ,       ):







      """simple docstring"""







      super().__init__(
          __lowercase       ,       __lowercase       ,       tokenizer_file=__lowercase       ,       errors=__lowercase       ,       bos_token=__lowercase       ,       eos_token=__lowercase       ,       sep_token=__lowercase       ,       cls_token=__lowercase       ,       unk_token=__lowercase       ,       pad_token=__lowercase       ,       mask_token=__lowercase       ,       add_prefix_space=__lowercase       ,       trim_offsets=__lowercase       ,       **__lowercase       ,       )

      UpperCamelCase         =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
      if pre_tok_state.get("""add_prefix_space"""       ,       __lowercase						) != add_prefix_space:
         UpperCamelCase         =					getattr(__lowercase       ,       pre_tok_state.pop("""type"""						)						)
         UpperCamelCase         =					add_prefix_space
         UpperCamelCase         =					pre_tok_class(**__lowercase						)

      UpperCamelCase         =					add_prefix_space

      UpperCamelCase         =					"""post_processor"""
      UpperCamelCase         =					getattr(self.backend_tokenizer       ,       __lowercase       ,       __lowercase						)
      if tokenizer_component_instance:
         UpperCamelCase         =					json.loads(tokenizer_component_instance.__getstate__()						)

         # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
         if "sep" in state:
            UpperCamelCase         =					tuple(state["""sep"""]						)
         if "cls" in state:
            UpperCamelCase         =					tuple(state["""cls"""]						)

         UpperCamelCase         =					False

         if state.get("""add_prefix_space"""       ,       __lowercase						) != add_prefix_space:
            UpperCamelCase         =					add_prefix_space
            UpperCamelCase         =					True

         if state.get("""trim_offsets"""       ,       __lowercase						) != trim_offsets:
            UpperCamelCase         =					trim_offsets
            UpperCamelCase         =					True

         if changes_to_apply:
            UpperCamelCase         =					getattr(__lowercase       ,       state.pop("""type"""						)						)
            UpperCamelCase         =					component_class(**__lowercase						)
            setattr(self.backend_tokenizer       ,       __lowercase       ,       __lowercase						)




   @property
   def   lowerCamelCase_					(						self							:		Optional[int]						):







      """simple docstring"""







      if self._mask_token is None:
         if self.verbose:
            logger.error("""Using mask_token, but it is not set yet."""						)
         return None
      return str(self._mask_token						)




   @mask_token.setter
   def   lowerCamelCase_					(						self							:		List[str]       ,       __magic_name__							:		Any						):







      """simple docstring"""







      UpperCamelCase         =					AddedToken(__lowercase       ,       lstrip=__lowercase       ,       rstrip=__lowercase						) if isinstance(__lowercase       ,       __lowercase						) else value
      UpperCamelCase         =					value




   def   lowerCamelCase_					(						self							:		List[Any]       ,       *__magic_name__							:		Tuple       ,       **__magic_name__							:		Any						):







      """simple docstring"""







      UpperCamelCase         =					kwargs.get("""is_split_into_words"""       ,       __lowercase						)
      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__lowercase       ,       **__lowercase						)




   def   lowerCamelCase_					(						self							:		List[Any]       ,       *__magic_name__							:		Tuple       ,       **__magic_name__							:		int						):







      """simple docstring"""







      UpperCamelCase         =					kwargs.get("""is_split_into_words"""       ,       __lowercase						)

      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__lowercase       ,       **__lowercase						)




   def   lowerCamelCase_					(						self							:		List[Any]       ,       __magic_name__							:		str       ,       __magic_name__							:		Optional[str] = None						):







      """simple docstring"""







      UpperCamelCase         =					self._tokenizer.model.save(__lowercase       ,       name=__lowercase						)
      return tuple(__lowercase						)




   def   lowerCamelCase_					(						self							:		Any       ,       __magic_name__							:		Tuple       ,       __magic_name__							:		Union[str, Any]=None						):







      """simple docstring"""







      UpperCamelCase         =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
      if token_ids_a is None:
         return output

      return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





   def   lowerCamelCase_					(						self							:		Dict       ,       __magic_name__							:		List[int]       ,       __magic_name__							:		Optional[List[int]] = None						):







      """simple docstring"""







      UpperCamelCase         =					[self.sep_token_id]
      UpperCamelCase         =					[self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep						) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]








import functools
import gc
import inspect

import torch

from .imports import is_npu_available, is_xpu_available






def 							_lowercase    (		*SCREAMING_SNAKE_CASE_   :		List[Any]	):





			"""simple docstring"""




			if not isinstance(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	):
						UpperCamelCase         =					list(SCREAMING_SNAKE_CASE_	)
			for i in range(len(SCREAMING_SNAKE_CASE_	)	):
						UpperCamelCase         =					None
			gc.collect()
			if is_xpu_available():
						torch.xpu.empty_cache()
			elif is_npu_available():
						torch.npu.empty_cache()
			else:
						torch.cuda.empty_cache()
			return objects






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		Exception	):





			"""simple docstring"""




			UpperCamelCase         =					[
			    """CUDA out of memory.""",  # CUDA OOM
			    """cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.""",  # CUDNN SNAFU
			    """DefaultCPUAllocator: can't allocate memory""",  # CPU OOM
			]
			if isinstance(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	) and len(exception.args	) == 1:
						return any(err in exception.args[0] for err in _statements	)
			return False






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		callable = None       ,  SCREAMING_SNAKE_CASE_   :		int = 128	):





			"""simple docstring"""




			if function is None:
						return functools.partial(SCREAMING_SNAKE_CASE_       ,  starting_batch_size=SCREAMING_SNAKE_CASE_	)

			UpperCamelCase         =					starting_batch_size

			def decorator(*SCREAMING_SNAKE_CASE_   :		Union[str, Any]       ,  **SCREAMING_SNAKE_CASE_   :		Tuple	):
						nonlocal batch_size
						gc.collect()
						if is_xpu_available():
									torch.xpu.empty_cache()
						elif is_npu_available():
									torch.npu.empty_cache()
						else:
									torch.cuda.empty_cache()
						UpperCamelCase         =					list(inspect.signature(SCREAMING_SNAKE_CASE_	).parameters.keys()	)
						# Guard against user error
						if len(SCREAMING_SNAKE_CASE_	) < (len(SCREAMING_SNAKE_CASE_	) + 1):
									UpperCamelCase         =					""", """.join([f'{arg}={value}' for arg, value in zip(params[1:]       ,  args[1:]	)]	)
									raise TypeError(
									    f'Batch size was passed into `{function.__name__}` as the first argument when called.'
									    f'Remove this as the decorator already does so: `{function.__name__}({arg_str})`'	)
						while True:
									if batch_size == 0:
												raise RuntimeError("""No executable batch size found, reached zero."""	)
									try:
												return function(SCREAMING_SNAKE_CASE_       ,  *SCREAMING_SNAKE_CASE_       ,  **SCREAMING_SNAKE_CASE_	)
									except Exception as e:
												if should_reduce_batch_size(SCREAMING_SNAKE_CASE_	):
															gc.collect()
															if is_xpu_available():
																		torch.xpu.empty_cache()
															elif is_npu_available():
																		torch.npu.empty_cache()
															else:
																		torch.cuda.empty_cache()
															batch_size //= 2
												else:
															raise

			return decorator




