





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


_UpperCAmelCase       : Dict			     =       logging.get_logger(__name__)

_UpperCAmelCase       : List[str]			     =       {"""vocab_file""": """vocab.txt"""}

_UpperCAmelCase       : List[Any]			     =       {
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

_UpperCAmelCase       : Dict			     =       {
    """YituTech/conv-bert-base""": 5_12,
    """YituTech/conv-bert-medium-small""": 5_12,
    """YituTech/conv-bert-small""": 5_12,
}


_UpperCAmelCase       : Optional[int]			     =       {
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}

class      lowerCAmelCase							(      __UpperCamelCase    ):
 UpperCAmelCase__									=	VOCAB_FILES_NAMES
 UpperCAmelCase__									=	PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__									=	PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__									=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__									=	ConvBertTokenizer
 def __init__(   self						:   Optional[int]    ,  UpperCAmelCase						:   Tuple=None    ,  UpperCAmelCase						:   List[str]=None    ,  UpperCAmelCase						:   Optional[Any]=True    ,  UpperCAmelCase						:   List[str]="[UNK]"    ,  UpperCAmelCase						:   Tuple="[SEP]"    ,  UpperCAmelCase						:   Any="[PAD]"    ,  UpperCAmelCase						:   str="[CLS]"    ,  UpperCAmelCase						:   Optional[Any]="[MASK]"    ,  UpperCAmelCase						:   List[str]=True    ,  UpperCAmelCase						:   Dict=None    ,  **UpperCAmelCase						:   str    ,  )    ->			List[Any]:
     super().__init__(
         UpperCAmelCase    ,  tokenizer_file=UpperCAmelCase    ,  do_lower_case=UpperCAmelCase    ,  unk_token=UpperCAmelCase    ,  sep_token=UpperCAmelCase    ,  pad_token=UpperCAmelCase    ,  cls_token=UpperCAmelCase    ,  mask_token=UpperCAmelCase    ,  tokenize_chinese_chars=UpperCAmelCase    ,  strip_accents=UpperCAmelCase    ,  **UpperCAmelCase    ,  )

     lowerCamelCase__					:   Any      				=	json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         normalizer_state.get('lowercase'    ,  UpperCAmelCase			) != do_lower_case
         or normalizer_state.get('strip_accents'    ,  UpperCAmelCase			) != strip_accents
         or normalizer_state.get('handle_chinese_chars'    ,  UpperCAmelCase			) != tokenize_chinese_chars
     ):
         lowerCamelCase__					:   List[str]      				=	getattr(UpperCAmelCase    ,  normalizer_state.pop('type'			)			)
         lowerCamelCase__					:   List[str]      				=	do_lower_case
         lowerCamelCase__					:   str      				=	strip_accents
         lowerCamelCase__					:   Optional[int]      				=	tokenize_chinese_chars
         lowerCamelCase__					:   Tuple      				=	normalizer_class(**UpperCAmelCase			)

     lowerCamelCase__					:   str      				=	do_lower_case
 def   A_			(   self						:   Optional[Any]    ,  UpperCAmelCase						:   List[Any]    ,  UpperCAmelCase						:   Optional[Any]=None			)    ->			str:
     lowerCamelCase__					:   str      				=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def   A_			(   self						:   str    ,  UpperCAmelCase						:   List[int]    ,  UpperCAmelCase						:   Optional[List[int]] = None			)    ->			List[int]:
     lowerCamelCase__					:   Union[str, Any]      				=	[self.sep_token_id]
     lowerCamelCase__					:   Optional[int]      				=	[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







 def   A_			(   self						:   Union[str, Any]    ,  UpperCAmelCase						:   str    ,  UpperCAmelCase						:   Optional[str] = None			)    ->			Tuple[str]:
     lowerCamelCase__					:   Union[str, Any]      				=	self._tokenizer.model.save(UpperCAmelCase    ,  name=UpperCAmelCase			)
     return tuple(UpperCAmelCase			)












from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase       : List[str]			     =       {
    """configuration_autoformer""": [
        """AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AutoformerConfig""",
    ],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : List[Any]			     =       [
        """AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """AutoformerForPrediction""",
        """AutoformerModel""",
        """AutoformerPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_autoformer import (
        AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
        AutoformerConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_autoformer import (
            AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            AutoformerForPrediction,
            AutoformerModel,
            AutoformerPreTrainedModel,
        )

else:
    import sys

    _UpperCAmelCase       : Dict			     =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





