
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase					:							str   =      logging.get_logger(__name__)

lowercase					:							Optional[Any]   =      {"vocab_file": "vocab.txt"}

lowercase					:							str   =      {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowercase					:							List[str]   =      {
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


lowercase					:							Optional[Any]   =      {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}


class 		__UpperCAmelCase   (      _lowerCamelCase		):
							__lowercase   	= VOCAB_FILES_NAMES
							__lowercase   	= PRETRAINED_VOCAB_FILES_MAP
							__lowercase   	= PRETRAINED_INIT_CONFIGURATION
							__lowercase   	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase   	= ConvBertTokenizer


							def __init__( self					,     lowerCAmelCase_=None					,     lowerCAmelCase_=None					,     lowerCAmelCase_=True					,     lowerCAmelCase_="[UNK]"					,     lowerCAmelCase_="[SEP]"					,     lowerCAmelCase_="[PAD]"					,     lowerCAmelCase_="[CLS]"					,     lowerCAmelCase_="[MASK]"					,     lowerCAmelCase_=True					,     lowerCAmelCase_=None					,     **lowerCAmelCase_					,     ):





								"""simple docstring"""






								super().__init__(
								    lowerCAmelCase_					,     tokenizer_file=lowerCAmelCase_					,     do_lower_case=lowerCAmelCase_					,     unk_token=lowerCAmelCase_					,     sep_token=lowerCAmelCase_					,     pad_token=lowerCAmelCase_					,     cls_token=lowerCAmelCase_					,     mask_token=lowerCAmelCase_					,     tokenize_chinese_chars=lowerCAmelCase_					,     strip_accents=lowerCAmelCase_					,     **lowerCAmelCase_					,     )

								_snake_case     		=	json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
								if (
								    normalizer_state.get('lowercase'					,     lowerCAmelCase_						) != do_lower_case
								    or normalizer_state.get('strip_accents'					,     lowerCAmelCase_						) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'					,     lowerCAmelCase_						) != tokenize_chinese_chars
								):
									_snake_case     		=	getattr(lowerCAmelCase_					,     normalizer_state.pop('type'						)						)
									_snake_case     		=	do_lower_case
									_snake_case     		=	strip_accents
									_snake_case     		=	tokenize_chinese_chars
									_snake_case     		=	normalizer_class(**lowerCAmelCase_						)

								_snake_case     		=	do_lower_case


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_=None						):





								"""simple docstring"""






								_snake_case     		=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	[self.sep_token_id]
								_snake_case     		=	[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	self._tokenizer.model.save(lowerCAmelCase_					,     name=lowerCAmelCase_						)
								return tuple(lowerCAmelCase_						)



'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowercase					:							str   =      {
    "configuration_nezha": ["NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP", "NezhaConfig"],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase					:							Any   =      [
					    "NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "NezhaForNextSentencePrediction",
					    "NezhaForMaskedLM",
					    "NezhaForPreTraining",
					    "NezhaForMultipleChoice",
					    "NezhaForQuestionAnswering",
					    "NezhaForSequenceClassification",
					    "NezhaForTokenClassification",
					    "NezhaModel",
					    "NezhaPreTrainedModel",
					]


if TYPE_CHECKING:
					from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_nezha import (
										    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
										    NezhaForMaskedLM,
										    NezhaForMultipleChoice,
										    NezhaForNextSentencePrediction,
										    NezhaForPreTraining,
										    NezhaForQuestionAnswering,
										    NezhaForSequenceClassification,
										    NezhaForTokenClassification,
										    NezhaModel,
										    NezhaPreTrainedModel,
										)


else:
					import sys

					lowercase					:							Union[str, Any]   =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

