






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_SCREAMING_SNAKE_CASE   :							Any          =		logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   :							Union[str, Any]          =		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE   :							Union[str, Any]          =		{
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		{
    '''bert-base-uncased''': 512,
    '''bert-large-uncased''': 512,
    '''bert-base-cased''': 512,
    '''bert-large-cased''': 512,
    '''bert-base-multilingual-uncased''': 512,
    '''bert-base-multilingual-cased''': 512,
    '''bert-base-chinese''': 512,
    '''bert-base-german-cased''': 512,
    '''bert-large-uncased-whole-word-masking''': 512,
    '''bert-large-cased-whole-word-masking''': 512,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 512,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 512,
    '''bert-base-cased-finetuned-mrpc''': 512,
    '''bert-base-german-dbmdz-cased''': 512,
    '''bert-base-german-dbmdz-uncased''': 512,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 512,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 512,
    '''wietsedv/bert-base-dutch-cased''': 512,
}

_SCREAMING_SNAKE_CASE   :							List[Any]          =		{
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}






class 	a		(   __snake_case						):
    SCREAMING_SNAKE_CASE		:	List[str]		=					VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		:	Any		=					PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		:	List[Any]		=					PRETRAINED_INIT_CONFIGURATION
    SCREAMING_SNAKE_CASE		:	List[Any]		=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		:	Any		=					BertTokenizer






    def __init__(			self	:				Optional[Any]			,		__SCREAMING_SNAKE_CASE	:				Tuple=None			,		__SCREAMING_SNAKE_CASE	:				Dict=None			,		__SCREAMING_SNAKE_CASE	:				Dict=True			,		__SCREAMING_SNAKE_CASE	:				Tuple="[UNK]"			,		__SCREAMING_SNAKE_CASE	:				List[str]="[SEP]"			,		__SCREAMING_SNAKE_CASE	:				int="[PAD]"			,		__SCREAMING_SNAKE_CASE	:				List[str]="[CLS]"			,		__SCREAMING_SNAKE_CASE	:				Optional[int]="[MASK]"			,		__SCREAMING_SNAKE_CASE	:				str=True			,		__SCREAMING_SNAKE_CASE	:				List[str]=None			,		**__SCREAMING_SNAKE_CASE	:				Any			,		)	->						List[Any]:
      super().__init__(
          __SCREAMING_SNAKE_CASE			,		tokenizer_file=__SCREAMING_SNAKE_CASE			,		do_lower_case=__SCREAMING_SNAKE_CASE			,		unk_token=__SCREAMING_SNAKE_CASE			,		sep_token=__SCREAMING_SNAKE_CASE			,		pad_token=__SCREAMING_SNAKE_CASE			,		cls_token=__SCREAMING_SNAKE_CASE			,		mask_token=__SCREAMING_SNAKE_CASE			,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE			,		strip_accents=__SCREAMING_SNAKE_CASE			,		**__SCREAMING_SNAKE_CASE			,		)

      lowerCamelCase_       	=		json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'			,		__SCREAMING_SNAKE_CASE   ) != do_lower_case
          or normalizer_state.get('strip_accents'			,		__SCREAMING_SNAKE_CASE   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'			,		__SCREAMING_SNAKE_CASE   ) != tokenize_chinese_chars
      ):
        lowerCamelCase_       	=		getattr(__SCREAMING_SNAKE_CASE			,		normalizer_state.pop('type'   )   )
        lowerCamelCase_       	=		do_lower_case
        lowerCamelCase_       	=		strip_accents
        lowerCamelCase_       	=		tokenize_chinese_chars
        lowerCamelCase_       	=		normalizer_class(**__SCREAMING_SNAKE_CASE   )

      lowerCamelCase_       	=		do_lower_case






    def   UpperCamelCase			(			self	:				List[str]			,		__SCREAMING_SNAKE_CASE	:				List[str]			,		__SCREAMING_SNAKE_CASE	:				Dict=None   )	->						Optional[int]:
      lowerCamelCase_       	=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output






    def   UpperCamelCase			(			self	:				Any			,		__SCREAMING_SNAKE_CASE	:				List[int]			,		__SCREAMING_SNAKE_CASE	:				Optional[List[int]] = None   )	->						List[int]:
      lowerCamelCase_       	=		[self.sep_token_id]
      lowerCamelCase_       	=		[self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def   UpperCamelCase			(			self	:				List[Any]			,		__SCREAMING_SNAKE_CASE	:				str			,		__SCREAMING_SNAKE_CASE	:				Optional[str] = None   )	->						Tuple[str]:
      lowerCamelCase_       	=		self._tokenizer.model.save(__SCREAMING_SNAKE_CASE			,		name=__SCREAMING_SNAKE_CASE   )
      return tuple(__SCREAMING_SNAKE_CASE   )












"""simple docstring"""





def 	lowerCamelCase__			(				_lowerCamelCase			:						str					) ->							bool:
  lowerCamelCase_       	=		0
  for ch in input_str:
    lowerCamelCase_       	=		ord(_lowerCamelCase					)
    lowerCamelCase_       	=		pow(2						,							_lowerCamelCase					)

    # If we already turned on bit for current character's unicode
    if bitmap >> ch_unicode & 1 == 1:
      return False
    bitmap |= ch_bit_index_on
  return True


if __name__ == "__main__":
  import doctest

  doctest.testmod()




