





"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCamelCase     :						Optional[int]						       =  logging.get_logger(__name__)

__UpperCamelCase     :						Optional[int]						       =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase     :						Dict						       =  {
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCamelCase     :						Union[str, Any]						       =  {
    '''distilbert-base-uncased''': 512,
    '''distilbert-base-uncased-distilled-squad''': 512,
    '''distilbert-base-cased''': 512,
    '''distilbert-base-cased-distilled-squad''': 512,
    '''distilbert-base-german-cased''': 512,
    '''distilbert-base-multilingual-cased''': 512,
}


__UpperCamelCase     :						Union[str, Any]						       =  {
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}







class       a							(   a__  ):
 snake_case__             =       VOCAB_FILES_NAMES
 snake_case__             =       PRETRAINED_VOCAB_FILES_MAP
 snake_case__             =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 snake_case__             =       PRETRAINED_INIT_CONFIGURATION
 snake_case__             =       ['''input_ids''', '''attention_mask''']
 snake_case__             =       DistilBertTokenizer



 def __init__(     self				,       _snake_case=None				,       _snake_case=None				,       _snake_case=True				,       _snake_case="[UNK]"				,       _snake_case="[SEP]"				,       _snake_case="[PAD]"				,       _snake_case="[CLS]"				,       _snake_case="[MASK]"				,       _snake_case=True				,       _snake_case=None				,       **_snake_case				,       ):







  """simple docstring"""






  super().__init__(
      _snake_case				,       tokenizer_file=_snake_case				,       do_lower_case=_snake_case				,       unk_token=_snake_case				,       sep_token=_snake_case				,       pad_token=_snake_case				,       cls_token=_snake_case				,       mask_token=_snake_case				,       tokenize_chinese_chars=_snake_case				,       strip_accents=_snake_case				,       **_snake_case				,       )

  lowerCAmelCase    				= json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
  if (
      normalizer_state.get('lowercase'				,       _snake_case     ) != do_lower_case
      or normalizer_state.get('strip_accents'				,       _snake_case     ) != strip_accents
      or normalizer_state.get('handle_chinese_chars'				,       _snake_case     ) != tokenize_chinese_chars
  ):
   lowerCAmelCase    				= getattr(_snake_case				,       normalizer_state.pop('type'     )     )
   lowerCAmelCase    				= do_lower_case
   lowerCAmelCase    				= strip_accents
   lowerCAmelCase    				= tokenize_chinese_chars
   lowerCAmelCase    				= normalizer_class(**_snake_case     )

  lowerCAmelCase    				= do_lower_case



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case=None     ):







  """simple docstring"""






  lowerCAmelCase    				= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

  if token_ids_a:
   output += token_ids_a + [self.sep_token_id]

  return output



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None     ):







  """simple docstring"""






  lowerCAmelCase    				= [self.sep_token_id]
  lowerCAmelCase    				= [self.cls_token_id]
  if token_ids_a is None:
   return len(cls + token_ids_a + sep     ) * [0]
  return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None     ):







  """simple docstring"""






  lowerCAmelCase    				= self._tokenizer.model.save(_snake_case				,       name=_snake_case     )
  return tuple(_snake_case     )








import argparse


_UpperCAmelCase						:  Union[str, Any]									=			"""docs/source/_static/js/custom.js"""





def    __lowerCamelCase      (					UpperCamelCase__ ):







							'''simple docstring'''






							with open(UpperCamelCase__  ,				encoding='utf-8'  ,				newline='\n' ) as f:
														snake_case_     							=       f.readlines()
							snake_case_     							=       0

							# First let's put the right version
							while not lines[index].startswith('const stableVersion =' ):
														index += 1
							snake_case_     							=       F'''const stableVersion = "v{version}"\n'''

							# Then update the dictionary
							while not lines[index].startswith('const versionMapping = {' ):
														index += 1

							# We go until the end
							while not lines[index].startswith('}' ):
														index += 1
							# We add the new version at the end
							lines[index - 1] += F'''    "v{version}": "v{version}",\n'''

							with open(UpperCamelCase__  ,				'w'  ,				encoding='utf-8'  ,				newline='\n' ) as f:
														f.writelines(UpperCamelCase__ )


if __name__ == "__main__":
					_UpperCAmelCase						:  int									=			argparse.ArgumentParser()
					parser.add_argument("""--version""", help="""Release version.""")
					_UpperCAmelCase						:  str									=			parser.parse_args()
					update_custom_js(args.version)




