




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


UpperCamelCase__					    =    logging.get_logger(__name__)

UpperCamelCase__					    =    {"""vocab_file""": """vocab.txt"""}

UpperCamelCase__					    =    {
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

UpperCamelCase__					    =    {
    """YituTech/conv-bert-base""": 512,
    """YituTech/conv-bert-medium-small""": 512,
    """YituTech/conv-bert-small""": 512,
}


UpperCamelCase__					    =    {
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}
class     a__					(      snake_case__   ):
						_a							:					Union[str, Any]      =					VOCAB_FILES_NAMES
						_a							:					Optional[Any]      =					PRETRAINED_VOCAB_FILES_MAP
						_a							:					Dict      =					PRETRAINED_INIT_CONFIGURATION
						_a							:					Dict      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:					List[Any]      =					ConvBertTokenizer






						def __init__(     self	,    _A=None	,    _A=None	,    _A=True	,    _A="[UNK]"	,    _A="[SEP]"	,    _A="[PAD]"	,    _A="[CLS]"	,    _A="[MASK]"	,    _A=True	,    _A=None	,    **_A	,    ):




											"""simple docstring"""



											super().__init__(
											    _A	,    tokenizer_file=_A	,    do_lower_case=_A	,    unk_token=_A	,    sep_token=_A	,    pad_token=_A	,    cls_token=_A	,    mask_token=_A	,    tokenize_chinese_chars=_A	,    strip_accents=_A	,    **_A	,    )

											__lowerCAmelCase            = json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
											if (
											    normalizer_state.get("lowercase"	,    _A    ) != do_lower_case
											    or normalizer_state.get("strip_accents"	,    _A    ) != strip_accents
											    or normalizer_state.get("handle_chinese_chars"	,    _A    ) != tokenize_chinese_chars
											):
																__lowerCAmelCase            = getattr(_A	,    normalizer_state.pop("type"    )    )
																__lowerCAmelCase            = do_lower_case
																__lowerCAmelCase            = strip_accents
																__lowerCAmelCase            = tokenize_chinese_chars
																__lowerCAmelCase            = normalizer_class(**_A    )

											__lowerCAmelCase            = do_lower_case






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A=None    ):




											"""simple docstring"""



											__lowerCAmelCase            = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

											return output






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = None    ):




											"""simple docstring"""



											__lowerCAmelCase            = [self.sep_token_id]
											__lowerCAmelCase            = [self.cls_token_id]
											if token_ids_a is None:
																return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = None    ):




											"""simple docstring"""



											__lowerCAmelCase            = self._tokenizer.model.save(_A	,    name=_A    )
											return tuple(_A    )










from typing import TYPE_CHECKING

from ....utils import _LazyModule


UpperCamelCase__					    =    {"""tokenization_tapex""": ["""TapexTokenizer"""]}


if TYPE_CHECKING:
					from .tokenization_tapex import TapexTokenizer


else:
					import sys

					UpperCamelCase__					    =    _LazyModule(__name__, globals()["""__file__"""], _import_structure)




