





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowercase    : Tuple    	=  logging.get_logger(__name__)

lowercase    : Union[str, Any]    	=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase    : str    	=  {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase    : Dict    	=  {
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


lowercase    : Optional[int]    	=  {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}

class  __A(						__UpperCAmelCase ):
     __A	  =       VOCAB_FILES_NAMES
     __A	  =       PRETRAINED_VOCAB_FILES_MAP
     __A	  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A	  =       PRETRAINED_INIT_CONFIGURATION
     __A	  =       ["input_ids", "attention_mask"]
     __A	  =       DistilBertTokenizer

     def __init__(      self,       A=None,       A=None,       A=True,       A="[UNK]",       A="[SEP]",       A="[PAD]",       A="[CLS]",       A="[MASK]",       A=True,       A=None,       **A,       ):





            """simple docstring"""

            super().__init__(
                A,       tokenizer_file=A,       do_lower_case=A,       unk_token=A,       sep_token=A,       pad_token=A,       cls_token=A,       mask_token=A,       tokenize_chinese_chars=A,       strip_accents=A,       **A,       )

            _UpperCamelCase              =						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
            if (
                normalizer_state.get('''lowercase''',       A      ) != do_lower_case
                or normalizer_state.get('''strip_accents''',       A      ) != strip_accents
                or normalizer_state.get('''handle_chinese_chars''',       A      ) != tokenize_chinese_chars
            ):
                   _UpperCamelCase              =						getattr(A,       normalizer_state.pop('''type'''      )      )
                   _UpperCamelCase              =						do_lower_case
                   _UpperCamelCase              =						strip_accents
                   _UpperCamelCase              =						tokenize_chinese_chars
                   _UpperCamelCase              =						normalizer_class(**A      )

            _UpperCamelCase              =						do_lower_case

     def 	_UpperCamelCase     (      self,       A,       A=None      ):





            """simple docstring"""

            _UpperCamelCase              =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

            return output

     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            _UpperCamelCase              =						[self.sep_token_id]
            _UpperCamelCase              =						[self.cls_token_id]
            if token_ids_a is None:
                   return len(cls + token_ids_a + sep      ) * [0]
            return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            _UpperCamelCase              =						self._tokenizer.model.save(A,       name=A      )
            return tuple(A      )













import warnings

from ...utils import logging
from .image_processing_clip import CLIPImageProcessor


lowercase    : str    	=  logging.get_logger(__name__)

class  __A(						__UpperCAmelCase ):

     def __init__(      self,       *A,       **A      ):





            """simple docstring"""

            warnings.warn(
                '''The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
                ''' use CLIPImageProcessor instead.''',       A,       )
            super().__init__(*A,       **A      )






