

'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


A_	= logging.get_logger(__name__)

A_	= {"vocab_file": "vocab.txt"}

A_	= {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

A_	= {
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


A_	= {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}


class  UpperCAmelCase (							UpperCAmelCase__    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE_           =					PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE_           =					PRETRAINED_INIT_CONFIGURATION
		SCREAMING_SNAKE_CASE_           =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE_           =					ConvBertTokenizer
		def __init__(					self   ,			SCREAMING_SNAKE_CASE_=None   ,			SCREAMING_SNAKE_CASE_=None   ,			SCREAMING_SNAKE_CASE_=True   ,			SCREAMING_SNAKE_CASE_="[UNK]"   ,			SCREAMING_SNAKE_CASE_="[SEP]"   ,			SCREAMING_SNAKE_CASE_="[PAD]"   ,			SCREAMING_SNAKE_CASE_="[CLS]"   ,			SCREAMING_SNAKE_CASE_="[MASK]"   ,			SCREAMING_SNAKE_CASE_=True   ,			SCREAMING_SNAKE_CASE_=None   ,			**SCREAMING_SNAKE_CASE_   ,			)  ->     int:






							'''simple docstring'''



							super().__init__(
							    SCREAMING_SNAKE_CASE_   ,			tokenizer_file=SCREAMING_SNAKE_CASE_   ,			do_lower_case=SCREAMING_SNAKE_CASE_   ,			unk_token=SCREAMING_SNAKE_CASE_   ,			sep_token=SCREAMING_SNAKE_CASE_   ,			pad_token=SCREAMING_SNAKE_CASE_   ,			cls_token=SCREAMING_SNAKE_CASE_   ,			mask_token=SCREAMING_SNAKE_CASE_   ,			tokenize_chinese_chars=SCREAMING_SNAKE_CASE_   ,			strip_accents=SCREAMING_SNAKE_CASE_   ,			**SCREAMING_SNAKE_CASE_   ,			)

							lowerCamelCase_  				=      json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
							if (
							    normalizer_state.get('lowercase'   ,			SCREAMING_SNAKE_CASE_		) != do_lower_case
							    or normalizer_state.get('strip_accents'   ,			SCREAMING_SNAKE_CASE_		) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'   ,			SCREAMING_SNAKE_CASE_		) != tokenize_chinese_chars
							):
												lowerCamelCase_  				=      getattr(SCREAMING_SNAKE_CASE_   ,			normalizer_state.pop('type'		)		)
												lowerCamelCase_  				=      do_lower_case
												lowerCamelCase_  				=      strip_accents
												lowerCamelCase_  				=      tokenize_chinese_chars
												lowerCamelCase_  				=      normalizer_class(**SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      do_lower_case
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_=None		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None		)  ->     List[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      [self.sep_token_id]
							lowerCamelCase_  				=      [self.cls_token_id]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None		)  ->     Tuple[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      self._tokenizer.model.save(SCREAMING_SNAKE_CASE_   ,			name=SCREAMING_SNAKE_CASE_		)
							return tuple(SCREAMING_SNAKE_CASE_		)






A_			:					Union[str, Any]    			=     '\n# Installazione di Transformers\n! pip install transformers datasets\n# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e\n# rimuovi la modalità commento al comando seguente.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

A_			:					str    			=     [{'type': 'code', 'content': INSTALL_CONTENT}]
A_			:					Any    			=     {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}



