






'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_a    :   str        =			logging.get_logger(__name__)

_a    :   Dict        =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_a    :   Union[str, Any]        =			{
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

_a    :   Optional[int]        =			{
    """bert-base-uncased""": 512,
    """bert-large-uncased""": 512,
    """bert-base-cased""": 512,
    """bert-large-cased""": 512,
    """bert-base-multilingual-uncased""": 512,
    """bert-base-multilingual-cased""": 512,
    """bert-base-chinese""": 512,
    """bert-base-german-cased""": 512,
    """bert-large-uncased-whole-word-masking""": 512,
    """bert-large-cased-whole-word-masking""": 512,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 512,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 512,
    """bert-base-cased-finetuned-mrpc""": 512,
    """bert-base-german-dbmdz-cased""": 512,
    """bert-base-german-dbmdz-uncased""": 512,
    """TurkuNLP/bert-base-finnish-cased-v1""": 512,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 512,
    """wietsedv/bert-base-dutch-cased""": 512,
}

_a    :   Union[str, Any]        =			{
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}







class 	_UpperCAmelCase  (  _A		):




      """simple docstring"""



      A          =    VOCAB_FILES_NAMES
      A          =    PRETRAINED_VOCAB_FILES_MAP
      A          =    PRETRAINED_INIT_CONFIGURATION
      A          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A          =    BertTokenizer
      def __init__(   self    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=True    ,	_lowerCAmelCase="[UNK]"    ,	_lowerCAmelCase="[SEP]"    ,	_lowerCAmelCase="[PAD]"    ,	_lowerCAmelCase="[CLS]"    ,	_lowerCAmelCase="[MASK]"    ,	_lowerCAmelCase=True    ,	_lowerCAmelCase=None    ,	**_lowerCAmelCase    ,	):



          '''simple docstring'''
          super().__init__(
              _lowerCAmelCase    ,	tokenizer_file=_lowerCAmelCase    ,	do_lower_case=_lowerCAmelCase    ,	unk_token=_lowerCAmelCase    ,	sep_token=_lowerCAmelCase    ,	pad_token=_lowerCAmelCase    ,	cls_token=_lowerCAmelCase    ,	mask_token=_lowerCAmelCase    ,	tokenize_chinese_chars=_lowerCAmelCase    ,	strip_accents=_lowerCAmelCase    ,	**_lowerCAmelCase    ,	)

          lowerCAmelCase__   :List[Any]      							=				json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("lowercase"    ,	_lowerCAmelCase		) != do_lower_case
              or normalizer_state.get("strip_accents"    ,	_lowerCAmelCase		) != strip_accents
              or normalizer_state.get("handle_chinese_chars"    ,	_lowerCAmelCase		) != tokenize_chinese_chars
          ):
              lowerCAmelCase__   :Union[str, Any]      							=				getattr(_lowerCAmelCase    ,	normalizer_state.pop("type"		)		)
              lowerCAmelCase__   :Tuple      							=				do_lower_case
              lowerCAmelCase__   :str      							=				strip_accents
              lowerCAmelCase__   :List[Any]      							=				tokenize_chinese_chars
              lowerCAmelCase__   :Optional[int]      							=				normalizer_class(**_lowerCAmelCase		)

          lowerCAmelCase__   :Any      							=				do_lower_case
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None		):



          '''simple docstring'''
          lowerCAmelCase__   :List[Any]      							=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''
          lowerCAmelCase__   :Any      							=				[self.sep_token_id]
          lowerCAmelCase__   :Any      							=				[self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]



      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''
          lowerCAmelCase__   :int      							=				self._tokenizer.model.save(_lowerCAmelCase    ,	name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)











from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

_a    :   Optional[Any]        =			[
    """python""",
    """tqdm""",
    """regex""",
    """requests""",
    """packaging""",
    """filelock""",
    """numpy""",
    """tokenizers""",
    """huggingface-hub""",
    """safetensors""",
    """accelerate""",
    """pyyaml""",
]

for pkg in pkgs_to_check_at_runtime:
 if pkg in deps:
  if pkg == "tokenizers":
   # must be loaded here, or else tqdm check may fail
   from .utils import is_tokenizers_available

   if not is_tokenizers_available():
    continue  # not required, check version only if installed
  elif pkg == "accelerate":
   # must be loaded here, or else tqdm check may fail
   from .utils import is_accelerate_available

   # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
   # Transformers with PyTorch
   if not is_accelerate_available():
    continue  # not required, check version only if installed

  require_version_core(deps[pkg])
 else:
  raise ValueError(f"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")


def 				snake_case__							(							UpperCAmelCase							:  Any    ,			UpperCAmelCase							:  Union[str, Any]=None       ):
    require_version(deps[pkg]    ,			UpperCAmelCase       )



