




'''simple docstring'''



import importlib.metadata
import operator
import re
import sys
from typing import Optional

from packaging import version


UpperCamelCase_    	=				{
    "<": operator.lt,
    "<=": operator.le,
    "==": operator.eq,
    "!=": operator.ne,
    ">=": operator.ge,
    ">": operator.gt,
}




def   lowercase__(			__UpperCamelCase:					Optional[Any]      ,__UpperCamelCase:					Optional[int]      ,__UpperCamelCase:					int      ,__UpperCamelCase:					Any      ,__UpperCamelCase:					str      ,__UpperCamelCase:					int    ):

      """simple docstring"""






      if got_ver is None or want_ver is None:
            raise ValueError(
                f"Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider"
                f" reinstalling {pkg}."    )
      if not ops[op](version.parse(__UpperCamelCase    )      ,version.parse(__UpperCamelCase    )    ):
            raise ImportError(
                f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}"    )




def   lowercase__(			__UpperCamelCase:					str      ,__UpperCamelCase:					Optional[str] = None    ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE   :      List[str]    					=						f"\n{hint}" if hint is not None else ''

      # non-versioned check
      if re.match(r'^[\w_\-\d]+$'      ,__UpperCamelCase    ):
            SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      Any    					=						requirement, None, None
      else:
            SCREAMING_SNAKE_CASE   :      List[Any]    					=						re.findall(r'^([^!=<>\s]+)([\s!=<>]{1,2}.+)'      ,__UpperCamelCase    )
            if not match:
                  raise ValueError(
                      'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but'
                      f" got {requirement}"    )
            SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      Tuple    					=						match[0]
            SCREAMING_SNAKE_CASE   :      Optional[int]    					=						want_full.split(','    )  # there could be multiple requirements
            SCREAMING_SNAKE_CASE   :      List[str]    					=						{}
            for w in want_range:
                  SCREAMING_SNAKE_CASE   :      List[Any]    					=						re.findall(r'^([\s!=<>]{1,2})(.+)'      ,__UpperCamelCase    )
                  if not match:
                        raise ValueError(
                            'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,'
                            f" but got {requirement}"    )
                  SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      Optional[int]    					=						match[0]
                  SCREAMING_SNAKE_CASE   :      Optional[int]    					=						want_ver
                  if op not in ops:
                        raise ValueError(f"{requirement}: need one of {list(ops.keys()    )}, but got {op}"    )

    # special case
      if pkg == "python":
            SCREAMING_SNAKE_CASE   :      Dict    					=						'.'.join([str(__UpperCamelCase    ) for x in sys.version_info[:3]]    )
            for op, want_ver in wanted.items():
                  _compare_versions(__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase    )
            return

      # check if any version is installed
      try:
            SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						importlib.metadata.version(__UpperCamelCase    )
      except importlib.metadata.PackageNotFoundError:
            raise importlib.metadata.PackageNotFoundError(
                f"The '{requirement}' distribution was not found and is required by this application. {hint}"    )

      # check that the right version is installed if version number or a range was provided
      if want_ver is not None:
            for op, want_ver in wanted.items():
                  _compare_versions(__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase      ,__UpperCamelCase    )





def   lowercase__(			__UpperCamelCase:					List[Any]    ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE   :      Tuple    					=						'Try: pip install transformers -U or pip install -e \'.[dev]\' if you\'re working with git main'
      return require_version(__UpperCamelCase      ,__UpperCamelCase    )








'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCamelCase_    	=				logging.get_logger(__name__)

UpperCamelCase_    	=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase_    	=				{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase_    	=				{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


UpperCamelCase_    	=				{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}






class 							_a				(			SCREAMING_SNAKE_CASE ):





       '''simple docstring'''






       A						:				List[Any]        =						VOCAB_FILES_NAMES
       A						:				Dict        =						PRETRAINED_VOCAB_FILES_MAP
       A						:				Optional[int]        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A						:				Optional[Any]        =						PRETRAINED_INIT_CONFIGURATION
       A						:				Optional[int]        =						['''input_ids''', '''attention_mask''']
       A						:				List[Any]        =						DistilBertTokenizer







       def __init__(      self,							A=None,							A=None,							A=True,							A="[UNK]",							A="[SEP]",							A="[PAD]",							A="[CLS]",							A="[MASK]",							A=True,							A=None,							**A,							):







             '''simple docstring'''







             super().__init__(
                 A,							tokenizer_file=A,							do_lower_case=A,							unk_token=A,							sep_token=A,							pad_token=A,							cls_token=A,							mask_token=A,							tokenize_chinese_chars=A,							strip_accents=A,							**A,							)

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
             if (
                 normalizer_state.get('lowercase',							A  ) != do_lower_case
                 or normalizer_state.get('strip_accents',							A  ) != strip_accents
                 or normalizer_state.get('handle_chinese_chars',							A  ) != tokenize_chinese_chars
             ):
                   SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						getattr(A,							normalizer_state.pop('type'  )  )
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						do_lower_case
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						strip_accents
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenize_chinese_chars
                   SCREAMING_SNAKE_CASE   :      Dict    					=						normalizer_class(**A  )

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						do_lower_case







       def        UpperCamelCase_							(      self,							A,							A=None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						[self.sep_token_id]
             SCREAMING_SNAKE_CASE   :      str    					=						[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls + token_ids_a + sep  ) * [0]
             return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						self._tokenizer.model.save(A,							name=A  )
             return tuple(A  )


