

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__UpperCAmelCase			=					logging.get_logger(__name__)

__UpperCAmelCase			=					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase			=					{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

__UpperCAmelCase			=					{
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}







class   __lowercase      (     __lowerCamelCase       ):
  snake_case_												=							VOCAB_FILES_NAMES
  snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
  snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_												=							["""input_ids""", """attention_mask"""]
  snake_case_												=							RobertaTokenizer

  def __init__(	self		:      List[str]       ,A		:      Optional[int]=None       ,A		:      Tuple=None       ,A		:      int=None       ,A		:      Tuple="replace"       ,A		:      Tuple="<s>"       ,A		:      Dict="</s>"       ,A		:      Optional[int]="</s>"       ,A		:      Dict="<s>"       ,A		:      List[Any]="<unk>"       ,A		:      Optional[Any]="<pad>"       ,A		:      List[str]="<mask>"       ,A		:      List[Any]=False       ,A		:      int=True       ,**A		:      Optional[int]       ,):






   '''simple docstring'''
   super().__init__(
       A       ,A       ,tokenizer_file=A       ,errors=A       ,bos_token=A       ,eos_token=A       ,sep_token=A       ,cls_token=A       ,unk_token=A       ,pad_token=A       ,mask_token=A       ,add_prefix_space=A       ,trim_offsets=A       ,**A       ,)

   UpperCAmelCase__       :	Any        =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
   if pre_tok_state.get("""add_prefix_space"""       ,A     ) != add_prefix_space:
    UpperCAmelCase__       :	Union[str, Any]        =       getattr(A       ,pre_tok_state.pop("""type"""     )     )
    UpperCAmelCase__       :	Dict        =       add_prefix_space
    UpperCAmelCase__       :	Dict        =       pre_tok_class(**A     )

   UpperCAmelCase__       :	List[Any]        =       add_prefix_space

   UpperCAmelCase__       :	Dict        =       """post_processor"""
   UpperCAmelCase__       :	Any        =       getattr(self.backend_tokenizer       ,A       ,A     )
   if tokenizer_component_instance:
    UpperCAmelCase__       :	int        =       json.loads(tokenizer_component_instance.__getstate__()     )

    # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
    if "sep" in state:
     UpperCAmelCase__       :	List[Any]        =       tuple(state["""sep"""]     )
    if "cls" in state:
     UpperCAmelCase__       :	List[str]        =       tuple(state["""cls"""]     )

    UpperCAmelCase__       :	Dict        =       False

    if state.get("""add_prefix_space"""       ,A     ) != add_prefix_space:
     UpperCAmelCase__       :	Tuple        =       add_prefix_space
     UpperCAmelCase__       :	str        =       True

    if state.get("""trim_offsets"""       ,A     ) != trim_offsets:
     UpperCAmelCase__       :	Tuple        =       trim_offsets
     UpperCAmelCase__       :	str        =       True

    if changes_to_apply:
     UpperCAmelCase__       :	List[str]        =       getattr(A       ,state.pop("""type"""     )     )
     UpperCAmelCase__       :	List[Any]        =       component_class(**A     )
     setattr(self.backend_tokenizer       ,A       ,A     )

  @property
  def  __lowercase							(	self		:      Dict     ):






   '''simple docstring'''
   if self._mask_token is None:
    if self.verbose:
     logger.error("""Using mask_token, but it is not set yet."""     )
    return None
   return str(self._mask_token     )

  @mask_token.setter
  def  __lowercase							(	self		:      Dict       ,A		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       AddedToken(A       ,lstrip=A       ,rstrip=A     ) if isinstance(A       ,A     ) else value
   UpperCAmelCase__       :	Union[str, Any]        =       value

  def  __lowercase							(	self		:      int       ,*A		:      Optional[Any]       ,**A		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       kwargs.get("""is_split_into_words"""       ,A     )
   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*A       ,**A     )

  def  __lowercase							(	self		:      Union[str, Any]       ,*A		:      Optional[Any]       ,**A		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       kwargs.get("""is_split_into_words"""       ,A     )

   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*A       ,**A     )

  def  __lowercase							(	self		:      List[str]       ,A		:      str       ,A		:      Optional[str] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       self._tokenizer.model.save(A       ,name=A     )
   return tuple(A     )

  def  __lowercase							(	self		:      Optional[int]       ,A		:      List[Any]       ,A		:      str=None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[str]        =       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
   if token_ids_a is None:
    return output

   return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




  def  __lowercase							(	self		:      List[str]       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[Any]        =       [self.sep_token_id]
   UpperCAmelCase__       :	Dict        =       [self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]










def        __lowerCAmelCase		(	__lowerCamelCase						:     int						)	->    bool:
					if number < 0:
										raise ValueError("""number must not be negative"""						)
					return number & (number - 1) == 0


if __name__ == "__main__":
				import doctest

				doctest.testmod()




