


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCamelCase     		=							logging.get_logger(__name__)

__UpperCamelCase     		=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCamelCase     		=							{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCamelCase     		=							{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


__UpperCamelCase     		=							{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}







class 							lowerCamelCase__						(				_SCREAMING_SNAKE_CASE   ):






			"""simple docstring"""






			_UpperCamelCase		:       List[str] 			=		VOCAB_FILES_NAMES
			_UpperCamelCase		:       Union[str, Any] 			=		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase		:       Dict 			=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase		:       Optional[int] 			=		PRETRAINED_INIT_CONFIGURATION
			_UpperCamelCase		:       Any 			=		['input_ids', 'attention_mask']
			_UpperCamelCase		:       Optional[int] 			=		DistilBertTokenizer




			def __init__(				self   ,    snake_case=None   ,    snake_case=None   ,    snake_case=True   ,    snake_case="[UNK]"   ,    snake_case="[SEP]"   ,    snake_case="[PAD]"   ,    snake_case="[CLS]"   ,    snake_case="[MASK]"   ,    snake_case=True   ,    snake_case=None   ,    **snake_case   ,    ):

						'''simple docstring'''


						super().__init__(
						    snake_case   ,    tokenizer_file=snake_case   ,    do_lower_case=snake_case   ,    unk_token=snake_case   ,    sep_token=snake_case   ,    pad_token=snake_case   ,    cls_token=snake_case   ,    mask_token=snake_case   ,    tokenize_chinese_chars=snake_case   ,    strip_accents=snake_case   ,    **snake_case   ,    )

						UpperCamelCase__       =							json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
						if (
						    normalizer_state.get("lowercase"   ,    snake_case  ) != do_lower_case
						    or normalizer_state.get("strip_accents"   ,    snake_case  ) != strip_accents
						    or normalizer_state.get("handle_chinese_chars"   ,    snake_case  ) != tokenize_chinese_chars
						):
									UpperCamelCase__       =							getattr(snake_case   ,    normalizer_state.pop("type"  )  )
									UpperCamelCase__       =							do_lower_case
									UpperCamelCase__       =							strip_accents
									UpperCamelCase__       =							tokenize_chinese_chars
									UpperCamelCase__       =							normalizer_class(**snake_case  )

						UpperCamelCase__       =							do_lower_case




			def 					snake_case__					(				self   ,    snake_case   ,    snake_case=None  ):

						'''simple docstring'''


						UpperCamelCase__       =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output




			def 					snake_case__					(				self   ,    snake_case   ,    snake_case = None  ):

						'''simple docstring'''


						UpperCamelCase__       =							[self.sep_token_id]
						UpperCamelCase__       =							[self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep  ) * [0]
						return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







			def 					snake_case__					(				self   ,    snake_case   ,    snake_case = None  ):

						'''simple docstring'''


						UpperCamelCase__       =							self._tokenizer.model.save(snake_case   ,    name=snake_case  )
						return tuple(snake_case  )






'''simple docstring'''






from ...utils import is_torch_available, is_transformers_available


if is_transformers_available() and is_torch_available():
  from .pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings, VQDiffusionPipeline



