


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


UpperCAmelCase      =   logging.get_logger(__name__)

UpperCAmelCase      =   {"vocab_file": "vocab.txt"}

UpperCAmelCase      =   {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

UpperCAmelCase      =   {
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


UpperCAmelCase      =   {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class lowerCAmelCase_						(			_SCREAMING_SNAKE_CASE	):

 '''simple docstring'''

 __snake_case												=  VOCAB_FILES_NAMES
 __snake_case												=  PRETRAINED_VOCAB_FILES_MAP
 __snake_case												=  PRETRAINED_INIT_CONFIGURATION
 __snake_case												=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case												=  ConvBertTokenizer



 def __init__(     self		,				_UpperCAmelCase=None		,				_UpperCAmelCase=None		,				_UpperCAmelCase=True		,				_UpperCAmelCase="[UNK]"		,				_UpperCAmelCase="[SEP]"		,				_UpperCAmelCase="[PAD]"		,				_UpperCAmelCase="[CLS]"		,				_UpperCAmelCase="[MASK]"		,				_UpperCAmelCase=True		,				_UpperCAmelCase=None		,				**_UpperCAmelCase		,				):
      super().__init__(
          _UpperCAmelCase		,				tokenizer_file=_UpperCAmelCase		,				do_lower_case=_UpperCAmelCase		,				unk_token=_UpperCAmelCase		,				sep_token=_UpperCAmelCase		,				pad_token=_UpperCAmelCase		,				cls_token=_UpperCAmelCase		,				mask_token=_UpperCAmelCase		,				tokenize_chinese_chars=_UpperCAmelCase		,				strip_accents=_UpperCAmelCase		,				**_UpperCAmelCase		,				)

      snake_case_	=				json.loads(self.backend_tokenizer.normalizer.__getstate__() )
      if (
          normalizer_state.get('''lowercase'''		,				_UpperCAmelCase ) != do_lower_case
          or normalizer_state.get('''strip_accents'''		,				_UpperCAmelCase ) != strip_accents
          or normalizer_state.get('''handle_chinese_chars'''		,				_UpperCAmelCase ) != tokenize_chinese_chars
      ):
           snake_case_	=				getattr(_UpperCAmelCase		,				normalizer_state.pop('''type''' ) )
           snake_case_	=				do_lower_case
           snake_case_	=				strip_accents
           snake_case_	=				tokenize_chinese_chars
           snake_case_	=				normalizer_class(**_UpperCAmelCase )

      snake_case_	=				do_lower_case



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase=None ):
      snake_case_	=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase = None ):
      snake_case_	=				[self.sep_token_id]
      snake_case_	=				[self.cls_token_id]
      if token_ids_a is None:
           return len(cls + token_ids_a + sep ) * [0]
      return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase = None ):
      snake_case_	=				self._tokenizer.model.save(_UpperCAmelCase		,				name=_UpperCAmelCase )
      return tuple(_UpperCAmelCase )



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase      =   {"""configuration_xglm""": ["""XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XGLMConfig"""]}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase      =   ["""XGLMTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase      =   ["""XGLMTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase      =   [
       """XGLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """XGLMForCausalLM""",
       """XGLMModel""",
       """XGLMPreTrainedModel""",
   ]


try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase      =   [
       """FlaxXGLMForCausalLM""",
       """FlaxXGLMModel""",
       """FlaxXGLMPreTrainedModel""",
   ]


try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase      =   [
       """TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """TFXGLMForCausalLM""",
       """TFXGLMModel""",
       """TFXGLMPreTrainedModel""",
   ]


if TYPE_CHECKING:
   from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_xglm import XGLMTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_xglm_fast import XGLMTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_xglm import (
          TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFXGLMForCausalLM,
          TFXGLMModel,
          TFXGLMPreTrainedModel,
      )


else:
   import sys

   UpperCAmelCase      =   _LazyModule(__name__, globals()["""__file__"""], _import_structure)