


'''simple docstring'''




from ..utils import DummyObject, requires_backends




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  Dict          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      List[Any]       ,   *SCREAMING_SNAKE_CASE_       :      Union[str, Any]       ,   **SCREAMING_SNAKE_CASE_       :      str      ) ->							Dict:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      List[str]       ,   **SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							Dict:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      Optional[int]       ,   *SCREAMING_SNAKE_CASE_       :      Dict       ,   **SCREAMING_SNAKE_CASE_       :      Dict      ) ->							Union[str, Any]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  Optional[int]          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      str       ,   *SCREAMING_SNAKE_CASE_       :      Any       ,   **SCREAMING_SNAKE_CASE_       :      Dict      ) ->							Dict:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      List[Any]       ,   **SCREAMING_SNAKE_CASE_       :      int      ) ->							Optional[Any]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      int       ,   **SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							int:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  Tuple          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      Dict       ,   *SCREAMING_SNAKE_CASE_       :      Union[str, Any]       ,   **SCREAMING_SNAKE_CASE_       :      str      ) ->							int:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      Dict       ,   *SCREAMING_SNAKE_CASE_       :      Tuple       ,   **SCREAMING_SNAKE_CASE_       :      int      ) ->							Optional[int]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      Optional[int]       ,   **SCREAMING_SNAKE_CASE_       :      Any      ) ->							Optional[int]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  List[Any]          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      int       ,   **SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							Dict:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      str       ,   *SCREAMING_SNAKE_CASE_       :      Optional[Any]       ,   **SCREAMING_SNAKE_CASE_       :      Dict      ) ->							Optional[int]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      str       ,   *SCREAMING_SNAKE_CASE_       :      Dict       ,   **SCREAMING_SNAKE_CASE_       :      Union[str, Any]      ) ->							Optional[Any]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  List[str]          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      Tuple       ,   *SCREAMING_SNAKE_CASE_       :      Tuple       ,   **SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							List[str]:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      int       ,   *SCREAMING_SNAKE_CASE_       :      str       ,   **SCREAMING_SNAKE_CASE_       :      str      ) ->							str:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      str       ,   *SCREAMING_SNAKE_CASE_       :      Dict       ,   **SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							Optional[int]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )




class 			_lowercase (	metaclass=__lowercase							):
   _SCREAMING_SNAKE_CASE  :  List[Any]          =	["torch", "transformers", "onnx"]
   def __init__(						self       :      Any       ,   *SCREAMING_SNAKE_CASE_       :      int       ,   **SCREAMING_SNAKE_CASE_       :      Optional[Any]      ) ->							str:
        requires_backends(self       ,   ['torch', 'transformers', 'onnx']      )
   @classmethod
   def a  (						cls       :      Optional[int]       ,   *SCREAMING_SNAKE_CASE_       :      List[Any]       ,   **SCREAMING_SNAKE_CASE_       :      str      ) ->							Dict:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )





   @classmethod
   def a  (						cls       :      Optional[Any]       ,   *SCREAMING_SNAKE_CASE_       :      Tuple       ,   **SCREAMING_SNAKE_CASE_       :      List[Any]      ) ->							Optional[int]:
        requires_backends(cls       ,   ['torch', 'transformers', 'onnx']      )








"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase_							=							logging.get_logger(__name__)

lowercase_							=							{'vocab_file': 'vocab.txt'}

lowercase_							=							{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

lowercase_							=							{
    'YituTech/conv-bert-base': 5_12,
    'YituTech/conv-bert-medium-small': 5_12,
    'YituTech/conv-bert-small': 5_12,
}


lowercase_							=							{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}







class    __a						( __snake_case							):
				lowerCamelCase     :       Optional[int]			       =VOCAB_FILES_NAMES
				lowerCamelCase     :       List[str]			       =PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase     :       str			       =PRETRAINED_INIT_CONFIGURATION
				lowerCamelCase     :       Any			       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase     :       List[str]			       =ConvBertTokenizer







				def __init__( self      ,	UpperCAmelCase=None      ,	UpperCAmelCase=None      ,	UpperCAmelCase=True      ,	UpperCAmelCase="[UNK]"      ,	UpperCAmelCase="[SEP]"      ,	UpperCAmelCase="[PAD]"      ,	UpperCAmelCase="[CLS]"      ,	UpperCAmelCase="[MASK]"      ,	UpperCAmelCase=True      ,	UpperCAmelCase=None      ,	**UpperCAmelCase      ,	):

					'''simple docstring'''






					super().__init__(
					    UpperCAmelCase      ,	tokenizer_file=UpperCAmelCase      ,	do_lower_case=UpperCAmelCase      ,	unk_token=UpperCAmelCase      ,	sep_token=UpperCAmelCase      ,	pad_token=UpperCAmelCase      ,	cls_token=UpperCAmelCase      ,	mask_token=UpperCAmelCase      ,	tokenize_chinese_chars=UpperCAmelCase      ,	strip_accents=UpperCAmelCase      ,	**UpperCAmelCase      ,	)

					lowerCAmelCase_      							=			json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
					if (
					    normalizer_state.get('''lowercase'''      ,	UpperCAmelCase   ) != do_lower_case
					    or normalizer_state.get('''strip_accents'''      ,	UpperCAmelCase   ) != strip_accents
					    or normalizer_state.get('''handle_chinese_chars'''      ,	UpperCAmelCase   ) != tokenize_chinese_chars
					):
						lowerCAmelCase_      							=			getattr(UpperCAmelCase      ,	normalizer_state.pop('''type'''   )   )
						lowerCAmelCase_      							=			do_lower_case
						lowerCAmelCase_      							=			strip_accents
						lowerCAmelCase_      							=			tokenize_chinese_chars
						lowerCAmelCase_      							=			normalizer_class(**UpperCAmelCase   )

					lowerCAmelCase_      							=			do_lower_case







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase=None   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			[self.sep_token_id]
					lowerCAmelCase_      							=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep   ) * [0]
					return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self._tokenizer.model.save(UpperCAmelCase      ,	name=UpperCAmelCase   )
					return tuple(UpperCAmelCase   )