


'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_UpperCamelCase 							=  logging.get_logger(__name__)

_UpperCamelCase 							=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase 							=  {
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCamelCase 							=  {
    """bert-base-uncased""": 512,
    """bert-large-uncased""": 512,
    """bert-base-cased""": 512,
    """bert-large-cased""": 512,
    """bert-base-multilingual-uncased""": 512,
    """bert-base-multilingual-cased""": 512,
    """bert-base-chinese""": 512,
    """bert-base-german-cased""": 512,
    """bert-large-uncased-whole-word-masking""": 512,
    """bert-large-cased-whole-word-masking""": 512,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 512,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 512,
    """bert-base-cased-finetuned-mrpc""": 512,
    """bert-base-german-dbmdz-cased""": 512,
    """bert-base-german-dbmdz-uncased""": 512,
    """TurkuNLP/bert-base-finnish-cased-v1""": 512,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 512,
    """wietsedv/bert-base-dutch-cased""": 512,
}

_UpperCamelCase 							=  {
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				VOCAB_FILES_NAMES
     lowerCamelCase__       						=				PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase__       						=				PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase__       						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase__       						=				BertTokenizer


     def __init__(   self			,     lowerCamelCase=None			,     lowerCamelCase=None			,     lowerCamelCase=True			,     lowerCamelCase="[UNK]"			,     lowerCamelCase="[SEP]"			,     lowerCamelCase="[PAD]"			,     lowerCamelCase="[CLS]"			,     lowerCamelCase="[MASK]"			,     lowerCamelCase=True			,     lowerCamelCase=None			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          super().__init__(
              lowerCamelCase			,     tokenizer_file=lowerCamelCase			,     do_lower_case=lowerCamelCase			,     unk_token=lowerCamelCase			,     sep_token=lowerCamelCase			,     pad_token=lowerCamelCase			,     cls_token=lowerCamelCase			,     mask_token=lowerCamelCase			,     tokenize_chinese_chars=lowerCamelCase			,     strip_accents=lowerCamelCase			,     **lowerCamelCase			,     )

          __A	:       List[str]      =							json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("lowercase"			,     lowerCamelCase		) != do_lower_case
              or normalizer_state.get("strip_accents"			,     lowerCamelCase		) != strip_accents
              or normalizer_state.get("handle_chinese_chars"			,     lowerCamelCase		) != tokenize_chinese_chars
          ):
               __A	:       Any      =							getattr(lowerCamelCase			,     normalizer_state.pop("type"		)		)
               __A	:       Optional[int]      =							do_lower_case
               __A	:       Optional[Any]      =							strip_accents
               __A	:       str      =							tokenize_chinese_chars
               __A	:       List[Any]      =							normalizer_class(**lowerCamelCase		)

          __A	:       List[str]      =							do_lower_case


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase=None		):


          '''simple docstring'''





          __A	:       Tuple      =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          __A	:       str      =							[self.sep_token_id]
          __A	:       Tuple      =							[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          __A	:       List[Any]      =							self._tokenizer.model.save(lowerCamelCase			,     name=lowerCamelCase		)
          return tuple(lowerCamelCase		)





'''simple docstring'''


import functools


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       int      =							len(SCREAMING_SNAKE_CASE							)
     __A	:       int      =							len(SCREAMING_SNAKE_CASE							)

     @functools.cache
     def min_distance(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							) -> int:
          # if first word index is overflow - delete all from the second word
          if indexa >= len_worda:
               return len_worda - indexa
          # if second word index is overflow - delete all from the first word
          if indexa >= len_worda:
               return len_worda - indexa
          __A	:       Optional[int]      =							int(worda[indexa] != worda[indexa]							)  # current letters not identical
          return min(
              1 + min_distance(indexa + 1       ,					SCREAMING_SNAKE_CASE							)       ,					1 + min_distance(SCREAMING_SNAKE_CASE       ,					indexa + 1							)       ,					diff + min_distance(indexa + 1       ,					indexa + 1							)       ,					)

     return min_distance(0       ,					0							)


if __name__ == "__main__":
      import doctest

      doctest.testmod()

