



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowerCAmelCase__              =							logging.get_logger(__name__)

lowerCAmelCase__              =							{'''vocab_file''': '''vocab.txt'''}

lowerCAmelCase__              =							{
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

lowerCAmelCase__              =							{
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


lowerCAmelCase__              =							{
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}






class 			__snake_case			(      _lowercase):
  snake_case__							:  List[str]         =		VOCAB_FILES_NAMES
  snake_case__							:  int         =		PRETRAINED_VOCAB_FILES_MAP
  snake_case__							:  List[str]         =		PRETRAINED_INIT_CONFIGURATION
  snake_case__							:  Union[str, Any]         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case__							:  Optional[int]         =		ConvBertTokenizer




  def __init__(					self	:		Dict		,   __lowerCAmelCase	:		Optional[int]=None		,   __lowerCAmelCase	:		Any=None		,   __lowerCAmelCase	:		Any=True		,   __lowerCAmelCase	:		Dict="[UNK]"		,   __lowerCAmelCase	:		Any="[SEP]"		,   __lowerCAmelCase	:		Dict="[PAD]"		,   __lowerCAmelCase	:		Any="[CLS]"		,   __lowerCAmelCase	:		Tuple="[MASK]"		,   __lowerCAmelCase	:		List[str]=True		,   __lowerCAmelCase	:		Dict=None		,   **__lowerCAmelCase	:		Optional[Any]		,   ):






         """simple docstring"""






         super().__init__(
             __lowerCAmelCase		,   tokenizer_file=__lowerCAmelCase		,   do_lower_case=__lowerCAmelCase		,   unk_token=__lowerCAmelCase		,   sep_token=__lowerCAmelCase		,   pad_token=__lowerCAmelCase		,   cls_token=__lowerCAmelCase		,   mask_token=__lowerCAmelCase		,   tokenize_chinese_chars=__lowerCAmelCase		,   strip_accents=__lowerCAmelCase		,   **__lowerCAmelCase		,   )

         _lowerCamelCase		:							Optional[Any]            =					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
         if (
             normalizer_state.get('''lowercase'''		,   __lowerCAmelCase						) != do_lower_case
             or normalizer_state.get('''strip_accents'''		,   __lowerCAmelCase						) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''		,   __lowerCAmelCase						) != tokenize_chinese_chars
         ):
                _lowerCamelCase		:							int            =					getattr(__lowerCAmelCase		,   normalizer_state.pop('''type'''						)						)
                _lowerCamelCase		:							Tuple            =					do_lower_case
                _lowerCamelCase		:							str            =					strip_accents
                _lowerCamelCase		:							List[str]            =					tokenize_chinese_chars
                _lowerCamelCase		:							Dict            =					normalizer_class(**__lowerCAmelCase						)

         _lowerCamelCase		:							str            =					do_lower_case




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[int]		,   __lowerCAmelCase	:		Any		,   __lowerCAmelCase	:		Union[str, Any]=None						):






         """simple docstring"""






         _lowerCamelCase		:							List[Any]            =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

         return output




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict		,   __lowerCAmelCase	:		List[int]		,   __lowerCAmelCase	:		Optional[List[int]] = None						):






         """simple docstring"""






         _lowerCamelCase		:							Optional[int]            =					[self.sep_token_id]
         _lowerCamelCase		:							Any            =					[self.cls_token_id]
         if token_ids_a is None:
                return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




  def 		SCREAMING_SNAKE_CASE							(					self	:		Union[str, Any]		,   __lowerCAmelCase	:		str		,   __lowerCAmelCase	:		Optional[str] = None						):






         """simple docstring"""






         _lowerCamelCase		:							List[Any]            =					self._tokenizer.model.save(__lowerCAmelCase		,   name=__lowerCAmelCase						)
         return tuple(__lowerCAmelCase						)




lowerCamelCase__     :		List[str]        = """
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
"""

lowerCamelCase__     :		List[Any]        = [{"""type""": """code""", """content""": INSTALL_CONTENT}]
lowerCamelCase__     :		int        = {
    """{processor_class}""": """FakeProcessorClass""",
    """{model_class}""": """FakeModelClass""",
    """{object_class}""": """FakeObjectClass""",
}






