

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__lowercase				      =						logging.get_logger(__name__)

__lowercase				      =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase				      =						{
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase				      =						{
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


__lowercase				      =						{
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}

class    _lowercase				(					__lowerCamelCase			):
   _lowercase     :       Optional[int]  					=    VOCAB_FILES_NAMES
   _lowercase     :       Any  					=    PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       Any  					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Optional[int]  					=    PRETRAINED_INIT_CONFIGURATION
   _lowercase     :       Union[str, Any]  					=    ['input_ids', 'attention_mask']
   _lowercase     :       str  					=    DistilBertTokenizer

   def __init__(				self	:    Optional[Any]			,						lowerCamelCase__	:    Optional[int]=None			,						lowerCamelCase__	:    str=None			,						lowerCamelCase__	:    Union[str, Any]=True			,						lowerCamelCase__	:    Optional[Any]="[UNK]"			,						lowerCamelCase__	:    List[str]="[SEP]"			,						lowerCamelCase__	:    Dict="[PAD]"			,						lowerCamelCase__	:    List[str]="[CLS]"			,						lowerCamelCase__	:    Optional[int]="[MASK]"			,						lowerCamelCase__	:    Union[str, Any]=True			,						lowerCamelCase__	:    Optional[Any]=None			,						**lowerCamelCase__	:    int			,						) ->			Dict:
         """simple docstring"""





         super().__init__(
             lowerCamelCase__			,						tokenizer_file=lowerCamelCase__			,						do_lower_case=lowerCamelCase__			,						unk_token=lowerCamelCase__			,						sep_token=lowerCamelCase__			,						pad_token=lowerCamelCase__			,						cls_token=lowerCamelCase__			,						mask_token=lowerCamelCase__			,						tokenize_chinese_chars=lowerCamelCase__			,						strip_accents=lowerCamelCase__			,						**lowerCamelCase__			,						)

         A_							       =		json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get('''lowercase'''			,						lowerCamelCase__       ) != do_lower_case
             or normalizer_state.get('''strip_accents'''			,						lowerCamelCase__       ) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''			,						lowerCamelCase__       ) != tokenize_chinese_chars
         ):
               A_							       =		getattr(lowerCamelCase__			,						normalizer_state.pop('''type'''       )       )
               A_							       =		do_lower_case
               A_							       =		strip_accents
               A_							       =		tokenize_chinese_chars
               A_							       =		normalizer_class(**lowerCamelCase__       )

         A_							       =		do_lower_case

   def     UpperCamelCase					(				self	:    Dict			,						lowerCamelCase__	:    Optional[int]			,						lowerCamelCase__	:    Dict=None       ) ->			int:
         """simple docstring"""





         A_							       =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output

   def     UpperCamelCase					(				self	:    int			,						lowerCamelCase__	:    List[int]			,						lowerCamelCase__	:    Optional[List[int]] = None       ) ->			List[int]:
         """simple docstring"""





         A_							       =		[self.sep_token_id]
         A_							       =		[self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





   def     UpperCamelCase					(				self	:    int			,						lowerCamelCase__	:    str			,						lowerCamelCase__	:    Optional[str] = None       ) ->			Tuple[str]:
         """simple docstring"""





         A_							       =		self._tokenizer.model.save(lowerCamelCase__			,						name=lowerCamelCase__       )
         return tuple(lowerCamelCase__       )








import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


__lowercase				      =						{
    """debug""": logging.DEBUG,
    """info""": logging.INFO,
    """warning""": logging.WARNING,
    """error""": logging.ERROR,
    """critical""": logging.CRITICAL,
}

__lowercase				      =						logging.WARNING







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		os.getenv('''DATASETS_VERBOSITY'''		,				SCREAMING_SNAKE_CASE   )
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      f"Unknown option DATASETS_VERBOSITY={env_level_str}, "
                      f"has to be one of: { ', '.join(log_levels.keys()   ) }"   )
      return _default_log_level







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return __name__.split('''.'''   )[0]







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return logging.getLogger(_get_library_name()   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		_get_library_root_logger()
      library_root_logger.setLevel(_get_default_logging_level()   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		_get_library_root_logger()
      library_root_logger.setLevel(logging.NOTSET   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE = None   ):
      '''simple docstring'''







      if name is None:
            A_							       =		_get_library_name()
      return logging.getLogger(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return _get_library_root_logger().getEffectiveLevel()







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      _get_library_root_logger().setLevel(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return set_verbosity(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return set_verbosity(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return set_verbosity(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      return set_verbosity(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		False







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()

class    _lowercase				:

   def __init__(				self	:    Optional[int]			,						*lowerCamelCase__	:    int			,						**lowerCamelCase__	:    Tuple       ) ->			Dict:  # pylint: disable=unused-argument
         """simple docstring"""





         A_							       =		args[0] if args else None

   def __iter__(				self	:    Optional[int]       ) ->			List[str]:
         """simple docstring"""





         return iter(self._iterator       )

   def __getattr__(				self	:    int			,						lowerCamelCase__	:    List[str]       ) ->			List[str]:
         """simple docstring"""





         def empty_fn(*lowerCamelCase__	:    Optional[Any]			,						**lowerCamelCase__	:    str       ):  # pylint: disable=unused-argument
               return

         return empty_fn

   def __enter__(				self	:    Tuple       ) ->			Tuple:
         """simple docstring"""





         return self





   def __exit__(				self	:    Tuple			,						lowerCamelCase__	:    List[Any]			,						lowerCamelCase__	:    List[str]			,						lowerCamelCase__	:    Optional[Any]       ) ->			Optional[int]:
         """simple docstring"""





         return



__lowercase				      =						True

class    _lowercase				:

   def __call__(				self	:    Optional[int]			,						*lowerCamelCase__	:    Optional[int]			,						lowerCamelCase__	:    List[str]=False			,						**lowerCamelCase__	:    Optional[Any]       ) ->			Optional[Any]:
         """simple docstring"""





         if _tqdm_active and not disable:
               return tqdm_lib.tqdm(*lowerCamelCase__			,						**lowerCamelCase__       )
         else:
               return EmptyTqdm(*lowerCamelCase__			,						**lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    Optional[Any]			,						*lowerCamelCase__	:    Any			,						**lowerCamelCase__	:    int       ) ->			str:
         """simple docstring"""





         A_							       =		None
         if _tqdm_active:
               return tqdm_lib.tqdm.set_lock(*lowerCamelCase__			,						**lowerCamelCase__       )





   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			int:
         """simple docstring"""





         if _tqdm_active:
               return tqdm_lib.tqdm.get_lock()



__lowercase				      =						_tqdm_cls()







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      global _tqdm_active
      return bool(_tqdm_active   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      global _tqdm_active
      A_							       =		True







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      global _tqdm_active
      A_							       =		False





