



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCAmelCase						:		Union[str, Any]			= logging.get_logger(__name__)

UpperCAmelCase						:		List[str]			= {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase						:		Tuple			= {
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase						:		Union[str, Any]			= {
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


UpperCAmelCase						:		List[str]			= {
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}


class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase   ):
    lowercase__      =   VOCAB_FILES_NAMES
    lowercase__      =   PRETRAINED_VOCAB_FILES_MAP
    lowercase__      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__      =   PRETRAINED_INIT_CONFIGURATION
    lowercase__      =   ["input_ids", "attention_mask"]
    lowercase__      =   DistilBertTokenizer

    def __init__(				self			:				int ,  lowerCAmelCase_			:				int=None ,  lowerCAmelCase_			:				str=None ,  lowerCAmelCase_			:				Optional[int]=True ,  lowerCAmelCase_			:				int="[UNK]" ,  lowerCAmelCase_			:				Any="[SEP]" ,  lowerCAmelCase_			:				Dict="[PAD]" ,  lowerCAmelCase_			:				Optional[Any]="[CLS]" ,  lowerCAmelCase_			:				int="[MASK]" ,  lowerCAmelCase_			:				List[Any]=True ,  lowerCAmelCase_			:				List[Any]=None ,  **lowerCAmelCase_			:				str ,  ):







      """simple docstring"""







      super().__init__(
          lowerCAmelCase_ ,  tokenizer_file=lowerCAmelCase_ ,  do_lower_case=lowerCAmelCase_ ,  unk_token=lowerCAmelCase_ ,  sep_token=lowerCAmelCase_ ,  pad_token=lowerCAmelCase_ ,  cls_token=lowerCAmelCase_ ,  mask_token=lowerCAmelCase_ ,  tokenize_chinese_chars=lowerCAmelCase_ ,  strip_accents=lowerCAmelCase_ ,  **lowerCAmelCase_ ,  )

      lowercase_								= json.loads(self.backend_tokenizer.normalizer.__getstate__())
      if (
          normalizer_state.get("""lowercase""" ,  lowerCAmelCase_) != do_lower_case
          or normalizer_state.get("""strip_accents""" ,  lowerCAmelCase_) != strip_accents
          or normalizer_state.get("""handle_chinese_chars""" ,  lowerCAmelCase_) != tokenize_chinese_chars
      ):
        lowercase_								= getattr(lowerCAmelCase_ ,  normalizer_state.pop("""type"""))
        lowercase_								= do_lower_case
        lowercase_								= strip_accents
        lowercase_								= tokenize_chinese_chars
        lowercase_								= normalizer_class(**lowerCAmelCase_)

      lowercase_								= do_lower_case

    def 					_UpperCAmelCase	(				self			:				Any ,  lowerCAmelCase_			:				Dict ,  lowerCAmelCase_			:				List[str]=None):







      """simple docstring"""







      lowercase_								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output

    def 					_UpperCAmelCase	(				self			:				Optional[Any] ,  lowerCAmelCase_			:				List[int] ,  lowerCAmelCase_			:				Optional[List[int]] = None):







      """simple docstring"""







      lowercase_								= [self.sep_token_id]
      lowercase_								= [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




    def 					_UpperCAmelCase	(				self			:				List[str] ,  lowerCAmelCase_			:				str ,  lowerCAmelCase_			:				Optional[str] = None):







      """simple docstring"""







      lowercase_								= self._tokenizer.model.save(lowerCAmelCase_ ,  name=lowerCAmelCase_)
      return tuple(lowerCAmelCase_)






"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import _LazyModule


UpperCAmelCase						:		List[Any]			= {"tokenization_bertweet": ["BertweetTokenizer"]}


if TYPE_CHECKING:
       from .tokenization_bertweet import BertweetTokenizer

else:
       import sys

       UpperCAmelCase						:		Tuple			= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

