






'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


UpperCamelCase     :   str	      =		logging.get_logger(__name__)

UpperCamelCase     :   Union[str, Any]	      =		{'vocab_file': 'vocab.txt'}

UpperCamelCase     :   int	      =		{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

UpperCamelCase     :   Tuple	      =		{
    'YituTech/conv-bert-base': 5_12,
    'YituTech/conv-bert-medium-small': 5_12,
    'YituTech/conv-bert-small': 5_12,
}


UpperCamelCase     :   Dict	      =		{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}




class 		UpperCamelCase__     (a      ):




		'''simple docstring'''





		_UpperCamelCase						=       VOCAB_FILES_NAMES
		_UpperCamelCase						=       PRETRAINED_VOCAB_FILES_MAP
		_UpperCamelCase						=       PRETRAINED_INIT_CONFIGURATION
		_UpperCamelCase						=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_UpperCamelCase						=       ConvBertTokenizer






		def __init__(   self ,_lowerCAmelCase=None ,_lowerCAmelCase=None ,_lowerCAmelCase=True ,_lowerCAmelCase="[UNK]" ,_lowerCAmelCase="[SEP]" ,_lowerCAmelCase="[PAD]" ,_lowerCAmelCase="[CLS]" ,_lowerCAmelCase="[MASK]" ,_lowerCAmelCase=True ,_lowerCAmelCase=None ,**_lowerCAmelCase ,):
								super().__init__(
								    _lowerCAmelCase ,tokenizer_file=_lowerCAmelCase ,do_lower_case=_lowerCAmelCase ,unk_token=_lowerCAmelCase ,sep_token=_lowerCAmelCase ,pad_token=_lowerCAmelCase ,cls_token=_lowerCAmelCase ,mask_token=_lowerCAmelCase ,tokenize_chinese_chars=_lowerCAmelCase ,strip_accents=_lowerCAmelCase ,**_lowerCAmelCase ,)

								lowerCamelCase__								=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
								if (
								    normalizer_state.get("""lowercase""" ,_lowerCAmelCase							) != do_lower_case
								    or normalizer_state.get("""strip_accents""" ,_lowerCAmelCase							) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars""" ,_lowerCAmelCase							) != tokenize_chinese_chars
								):
														lowerCamelCase__								=	getattr(_lowerCAmelCase ,normalizer_state.pop("""type"""							)							)
														lowerCamelCase__								=	do_lower_case
														lowerCamelCase__								=	strip_accents
														lowerCamelCase__								=	tokenize_chinese_chars
														lowerCamelCase__								=	normalizer_class(**_lowerCAmelCase							)

								lowerCamelCase__								=	do_lower_case






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase=None							):
								lowerCamelCase__								=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

								return output






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase = None							):
								lowerCamelCase__								=	[self.sep_token_id]
								lowerCamelCase__								=	[self.cls_token_id]
								if token_ids_a is None:
														return len(cls + token_ids_a + sep							) * [0]
								return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase = None							):
								lowerCamelCase__								=	self._tokenizer.model.save(_lowerCAmelCase ,name=_lowerCAmelCase							)
								return tuple(_lowerCAmelCase							)









"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_SCREAMING_SNAKE_CASE   :							List[Any]          =		{'''configuration_plbart''': ['''PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''PLBartConfig''']}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _SCREAMING_SNAKE_CASE   :							Union[str, Any]          =		['''PLBartTokenizer''']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _SCREAMING_SNAKE_CASE   :							List[str]          =		[
      '''PLBART_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''PLBartForCausalLM''',
      '''PLBartForConditionalGeneration''',
      '''PLBartForSequenceClassification''',
      '''PLBartModel''',
      '''PLBartPreTrainedModel''',
  ]


if TYPE_CHECKING:
  from .configuration_plbart import PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP, PLBartConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_plbart import PLBartTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_plbart import (
        PLBART_PRETRAINED_MODEL_ARCHIVE_LIST,
        PLBartForCausalLM,
        PLBartForConditionalGeneration,
        PLBartForSequenceClassification,
        PLBartModel,
        PLBartPreTrainedModel,
    )


else:
  import sys

  _SCREAMING_SNAKE_CASE   :							Optional[Any]          =		_LazyModule(__name__, globals()['''__file__'''], _import_structure)




