





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCAmelCase_							=				logging.get_logger(__name__)

UpperCAmelCase_							=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_							=				{
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase_							=				{
    """bert-base-uncased""": 512,
    """bert-large-uncased""": 512,
    """bert-base-cased""": 512,
    """bert-large-cased""": 512,
    """bert-base-multilingual-uncased""": 512,
    """bert-base-multilingual-cased""": 512,
    """bert-base-chinese""": 512,
    """bert-base-german-cased""": 512,
    """bert-large-uncased-whole-word-masking""": 512,
    """bert-large-cased-whole-word-masking""": 512,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 512,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 512,
    """bert-base-cased-finetuned-mrpc""": 512,
    """bert-base-german-dbmdz-cased""": 512,
    """bert-base-german-dbmdz-uncased""": 512,
    """TurkuNLP/bert-base-finnish-cased-v1""": 512,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 512,
    """wietsedv/bert-base-dutch-cased""": 512,
}

UpperCAmelCase_							=				{
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}
class     UpperCamelCase__					(     snake_case_				):






		'''simple docstring'''






		__a       :       Any       =	VOCAB_FILES_NAMES
		__a       :       str       =	PRETRAINED_VOCAB_FILES_MAP
		__a       :       int       =	PRETRAINED_INIT_CONFIGURATION
		__a       :       int       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__a       :       Optional[int]       =	BertTokenizer






		def __init__(							self,   snake_case__=None,   snake_case__=None,   snake_case__=True,   snake_case__="[UNK]",   snake_case__="[SEP]",   snake_case__="[PAD]",   snake_case__="[CLS]",   snake_case__="[MASK]",   snake_case__=True,   snake_case__=None,   **snake_case__,   )       -> int:






									"""simple docstring"""
									super().__init__(
									    snake_case__,   tokenizer_file=snake_case__,   do_lower_case=snake_case__,   unk_token=snake_case__,   sep_token=snake_case__,   pad_token=snake_case__,   cls_token=snake_case__,   mask_token=snake_case__,   tokenize_chinese_chars=snake_case__,   strip_accents=snake_case__,   **snake_case__,   )

									lowercase_	:   int  				=     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
									if (
									    normalizer_state.get("""lowercase""",   snake_case__  ) != do_lower_case
									    or normalizer_state.get("""strip_accents""",   snake_case__  ) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars""",   snake_case__  ) != tokenize_chinese_chars
									):
																lowercase_	:   Union[str, Any]  				=     getattr(snake_case__,   normalizer_state.pop("""type"""  )  )
																lowercase_	:   Union[str, Any]  				=     do_lower_case
																lowercase_	:   Optional[int]  				=     strip_accents
																lowercase_	:   Dict  				=     tokenize_chinese_chars
																lowercase_	:   Optional[Any]  				=     normalizer_class(**snake_case__  )

									lowercase_	:   List[str]  				=     do_lower_case






		def 						snake_case__				(							self,   snake_case__,   snake_case__=None  )       -> Tuple:






									"""simple docstring"""
									lowercase_	:   str  				=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

									return output






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None  )       -> Optional[Any]:






									"""simple docstring"""
									lowercase_	:   Dict  				=     [self.sep_token_id]
									lowercase_	:   List[str]  				=     [self.cls_token_id]
									if token_ids_a is None:
																return len(cls + token_ids_a + sep  ) * [0]
									return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None  )       -> Tuple:






									"""simple docstring"""
									lowercase_	:   Union[str, Any]  				=     self._tokenizer.model.save(snake_case__,   name=snake_case__  )
									return tuple(snake_case__  )
"""simple docstring"""



def 	_lowerCamelCase(	a = 1   ,    a = 1_0_0_0				):
       __a       =    1
       __a       =    0

       for divide_by_number in range(a   ,    digit + 1				):
              __a       =    []
              __a       =    numerator
              for _ in range(1   ,    digit + 1				):
                     if now_divide in has_been_divided:
                            if longest_list_length < len(a				):
                                   __a       =    len(a				)
                                   __a       =    divide_by_number
                     else:
                            has_been_divided.append(a				)
                            __a       =    now_divide * 1_0 % divide_by_number

       return the_digit


# Tests
if __name__ == "__main__":
   import doctest

   doctest.testmod()
