



'''simple docstring'''



import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    GPTNeoXForCausalLM,
							    GPTNeoXForQuestionAnswering,
							    GPTNeoXForSequenceClassification,
							    GPTNeoXForTokenClassification,
							    GPTNeoXModel,
							)
class 						lowerCamelCase_				:







						def __init__(	self :   Optional[int]					,     lowerCAmelCase__ :   Union[str, Any]					,     lowerCAmelCase__ :   Optional[int]=13					,     lowerCAmelCase__ :   List[str]=7					,     lowerCAmelCase__ :   Union[str, Any]=True					,     lowerCAmelCase__ :   Dict=True					,     lowerCAmelCase__ :   Any=True					,     lowerCAmelCase__ :   Optional[int]=True					,     lowerCAmelCase__ :   Dict=99					,     lowerCAmelCase__ :   List[str]=64					,     lowerCAmelCase__ :   Optional[Any]=5					,     lowerCAmelCase__ :   Any=4					,     lowerCAmelCase__ :   Optional[int]=37					,     lowerCAmelCase__ :   Tuple="gelu"					,     lowerCAmelCase__ :   Optional[int]=0.1					,     lowerCAmelCase__ :   Union[str, Any]=0.1					,     lowerCAmelCase__ :   int=5_12					,     lowerCAmelCase__ :   Optional[Any]=16					,     lowerCAmelCase__ :   List[Any]=2					,     lowerCAmelCase__ :   Any=0.02					,     lowerCAmelCase__ :   Optional[Any]=3					,     lowerCAmelCase__ :   str=4					,     lowerCAmelCase__ :   Union[str, Any]=None					,     ):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      parent
													SCREAMING_SNAKE_CASE       :   Dict         =      batch_size
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      seq_length
													SCREAMING_SNAKE_CASE       :   List[Any]         =      is_training
													SCREAMING_SNAKE_CASE       :   int         =      use_input_mask
													SCREAMING_SNAKE_CASE       :   List[str]         =      use_token_type_ids
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      use_labels
													SCREAMING_SNAKE_CASE       :   Tuple         =      vocab_size
													SCREAMING_SNAKE_CASE       :   int         =      hidden_size
													SCREAMING_SNAKE_CASE       :   str         =      num_hidden_layers
													SCREAMING_SNAKE_CASE       :   List[str]         =      num_attention_heads
													SCREAMING_SNAKE_CASE       :   List[Any]         =      intermediate_size
													SCREAMING_SNAKE_CASE       :   List[Any]         =      hidden_act
													SCREAMING_SNAKE_CASE       :   str         =      hidden_dropout_prob
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      attention_probs_dropout_prob
													SCREAMING_SNAKE_CASE       :   List[Any]         =      max_position_embeddings
													SCREAMING_SNAKE_CASE       :   int         =      type_vocab_size
													SCREAMING_SNAKE_CASE       :   str         =      type_sequence_label_size
													SCREAMING_SNAKE_CASE       :   Tuple         =      initializer_range
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      num_labels
													SCREAMING_SNAKE_CASE       :   List[str]         =      num_choices
													SCREAMING_SNAKE_CASE       :   List[str]         =      scope
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      vocab_size - 1







						def        __lowercase     (	self :   Optional[Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Any         =      ids_tensor([self.batch_size, self.seq_length]					,     self.vocab_size					)

													SCREAMING_SNAKE_CASE       :   str         =      None
													if self.use_input_mask:
																				SCREAMING_SNAKE_CASE       :   Any         =      random_attention_mask([self.batch_size, self.seq_length]					)

													SCREAMING_SNAKE_CASE       :   int         =      None
													if self.use_labels:
																				SCREAMING_SNAKE_CASE       :   Optional[int]         =      ids_tensor([self.batch_size, self.seq_length]					,     self.num_labels					)

													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.get_config()

													return config, input_ids, input_mask, token_labels







						def        __lowercase     (	self :   List[Any]					):

													"""simple docstring"""
													return GPTNeoXConfig(
													    vocab_size=self.vocab_size					,     hidden_size=self.hidden_size					,     num_hidden_layers=self.num_hidden_layers					,     num_attention_heads=self.num_attention_heads					,     intermediate_size=self.intermediate_size					,     hidden_act=self.hidden_act					,     hidden_dropout_prob=self.hidden_dropout_prob					,     attention_probs_dropout_prob=self.attention_probs_dropout_prob					,     max_position_embeddings=self.max_position_embeddings					,     type_vocab_size=self.type_vocab_size					,     is_decoder=lowercase_					,     initializer_range=self.initializer_range					,     pad_token_id=self.pad_token_id					,     )







						def        __lowercase     (	self :   List[str]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   int         =      self.prepare_config_and_inputs()

													SCREAMING_SNAKE_CASE       :   List[Any]         =      True

													return config, input_ids, input_mask, token_labels







						def        __lowercase     (	self :   str					,     lowerCAmelCase__ :   Tuple					,     lowerCAmelCase__ :   Optional[Any]					,     lowerCAmelCase__ :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Any         =      GPTNeoXModel(config=lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      model(lowercase_					,     attention_mask=lowercase_					)
													SCREAMING_SNAKE_CASE       :   List[Any]         =      model(lowercase_					)
													self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)					)







						def        __lowercase     (	self :   Union[str, Any]					,     lowerCAmelCase__ :   List[Any]					,     lowerCAmelCase__ :   List[str]					,     lowerCAmelCase__ :   Optional[Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   str         =      True
													SCREAMING_SNAKE_CASE       :   Tuple         =      GPTNeoXModel(lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   int         =      model(lowercase_					,     attention_mask=lowercase_					)
													self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)					)







						def        __lowercase     (	self :   Union[str, Any]					,     lowerCAmelCase__ :   Optional[int]					,     lowerCAmelCase__ :   Optional[Any]					,     lowerCAmelCase__ :   Tuple					,     lowerCAmelCase__ :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      GPTNeoXForCausalLM(config=lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   int         =      model(lowercase_					,     attention_mask=lowercase_					,     labels=lowercase_					)
													self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.vocab_size)					)







						def        __lowercase     (	self :   List[str]					,     lowerCAmelCase__ :   Any					,     lowerCAmelCase__ :   Any					,     lowerCAmelCase__ :   int					,     lowerCAmelCase__ :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      self.num_labels
													SCREAMING_SNAKE_CASE       :   int         =      GPTNeoXForQuestionAnswering(lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   Tuple         =      model(lowercase_					,     attention_mask=lowercase_					)
													self.parent.assertEqual(result.start_logits.shape					,     (self.batch_size, self.seq_length)					)
													self.parent.assertEqual(result.end_logits.shape					,     (self.batch_size, self.seq_length)					)







						def        __lowercase     (	self :   List[str]					,     lowerCAmelCase__ :   Optional[int]					,     lowerCAmelCase__ :   Any					,     lowerCAmelCase__ :   Tuple					,     lowerCAmelCase__ :   Dict					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Dict         =      self.num_labels
													SCREAMING_SNAKE_CASE       :   List[str]         =      GPTNeoXForSequenceClassification(lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   List[Any]         =      ids_tensor([self.batch_size]					,     self.type_sequence_label_size					)
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      model(lowercase_					,     attention_mask=lowercase_					,     labels=lowercase_					)
													self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.num_labels)					)







						def        __lowercase     (	self :   Optional[Any]					,     lowerCAmelCase__ :   Dict					,     lowerCAmelCase__ :   str					,     lowerCAmelCase__ :   Any					,     lowerCAmelCase__ :   Tuple					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      self.num_labels
													SCREAMING_SNAKE_CASE       :   Dict         =      GPTNeoXForTokenClassification(lowercase_					)
													model.to(lowercase_					)
													model.eval()
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      model(lowercase_					,     attention_mask=lowercase_					,     labels=lowercase_					)
													self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.num_labels)					)







						def        __lowercase     (	self :   Dict					,     lowerCAmelCase__ :   Dict					,     lowerCAmelCase__ :   Tuple					,     lowerCAmelCase__ :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      True
													SCREAMING_SNAKE_CASE       :   Tuple         =      GPTNeoXForCausalLM(config=lowercase_					)
													model.to(lowercase_					)
													model.eval()

													# first forward pass
													SCREAMING_SNAKE_CASE       :   List[Any]         =      model(lowercase_					,     attention_mask=lowercase_					,     use_cache=lowercase_					)
													SCREAMING_SNAKE_CASE       :   int         =      outputs.past_key_values

													# create hypothetical multiple next token and extent to next_input_ids
													SCREAMING_SNAKE_CASE       :   str         =      ids_tensor((self.batch_size, 3)					,     config.vocab_size					)
													SCREAMING_SNAKE_CASE       :   Tuple         =      ids_tensor((self.batch_size, 3)					,     vocab_size=2					)

													# append to next input_ids and
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      torch.cat([input_ids, next_tokens]					,     dim=-1					)
													SCREAMING_SNAKE_CASE       :   int         =      torch.cat([input_mask, next_mask]					,     dim=-1					)

													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      model(lowercase_					,     attention_mask=lowercase_					,     output_hidden_states=lowercase_					)
													SCREAMING_SNAKE_CASE       :   int         =      output_from_no_past['''hidden_states'''][0]
													SCREAMING_SNAKE_CASE       :   Any         =      model(
													    lowercase_					,     attention_mask=lowercase_					,     past_key_values=lowercase_					,     output_hidden_states=lowercase_					,     )['''hidden_states'''][0]

													# select random slice
													SCREAMING_SNAKE_CASE       :   Any         =      ids_tensor((1,)					,     output_from_past.shape[-1]					).item()
													SCREAMING_SNAKE_CASE       :   int         =      output_from_no_past[:, -3:, random_slice_idx].detach()
													SCREAMING_SNAKE_CASE       :   Dict         =      output_from_past[:, :, random_slice_idx].detach()

													self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

													# test that outputs are equal for slice
													self.parent.assertTrue(torch.allclose(lowercase_					,     lowercase_					,     atol=1e-3					)					)







						def        __lowercase     (	self :   Optional[Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[Any]         =      self.prepare_config_and_inputs()
													SCREAMING_SNAKE_CASE       :   Dict         =      config_and_inputs
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      {'''input_ids''': input_ids, '''attention_mask''': input_mask}
													return config, inputs_dict


@require_torch
class 						lowerCamelCase_				(       a_     ,					a_     ,					a_     ,					unittest.TestCase      ):
						_lowerCAmelCase  : Optional[Any]						  =					(
						    (
						        GPTNeoXModel,
						        GPTNeoXForCausalLM,
						        GPTNeoXForQuestionAnswering,
						        GPTNeoXForSequenceClassification,
						        GPTNeoXForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowerCAmelCase  : str						  =					(GPTNeoXForCausalLM,) if is_torch_available() else ()
						_lowerCAmelCase  : Any						  =					(
						    {
						        'feature-extraction': GPTNeoXModel,
						        'question-answering': GPTNeoXForQuestionAnswering,
						        'text-classification': GPTNeoXForSequenceClassification,
						        'text-generation': GPTNeoXForCausalLM,
						        'token-classification': GPTNeoXForTokenClassification,
						        'zero-shot': GPTNeoXForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_lowerCAmelCase  : Dict						  =					False
						_lowerCAmelCase  : Tuple						  =					False
						_lowerCAmelCase  : Optional[Any]						  =					False
						_lowerCAmelCase  : int						  =					False







						def        __lowercase     (	self :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      GPTNeoXModelTester(self					)
													SCREAMING_SNAKE_CASE       :   Any         =      ConfigTester(self					,     config_class=lowercase_					,     hidden_size=64					,     num_attention_heads=8					)







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													self.config_tester.run_common_tests()







						def        __lowercase     (	self :   Any					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[Any]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(lowercase_					,     lowercase_					,     lowercase_					)







						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      self.model_tester.prepare_config_and_inputs_for_decoder()
													self.model_tester.create_and_check_model_as_decoder(lowercase_					,     lowercase_					,     lowercase_					)







						def        __lowercase     (	self :   Optional[int]					):

													"""simple docstring"""
													# This regression test was failing with PyTorch < 1.3
													SCREAMING_SNAKE_CASE       :   Any         =      self.model_tester.prepare_config_and_inputs_for_decoder()

													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      None

													self.model_tester.create_and_check_model_as_decoder(lowercase_					,     lowercase_					,     lowercase_					)







						def        __lowercase     (	self :   List[Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_decoder_model_past_large_inputs(lowercase_					,     lowercase_					,     lowercase_					)







						def        __lowercase     (	self :   Optional[Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[Any]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_causal_lm(*lowercase_					)







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*lowercase_					)







						def        __lowercase     (	self :   List[str]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*lowercase_					)







						def        __lowercase     (	self :   List[str]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*lowercase_					)







						@unittest.skip(reason='''Feed forward chunking is not implemented'''					)
						def        __lowercase     (	self :   Optional[Any]					):

													"""simple docstring"""
													pass







						@parameterized.expand([('''linear''',), ('''dynamic''',)]					)
						def        __lowercase     (	self :   Any					,     lowerCAmelCase__ :   Optional[int]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      self.model_tester.prepare_config_and_inputs_for_common()
													SCREAMING_SNAKE_CASE       :   Any         =      ids_tensor([1, 10]					,     config.vocab_size					)
													SCREAMING_SNAKE_CASE       :   Any         =      ids_tensor([1, int(config.max_position_embeddings * 1.5					)]					,     config.vocab_size					)

													set_seed(42					)  # Fixed seed at init time so the two models get the same random weights
													SCREAMING_SNAKE_CASE       :   List[Any]         =      GPTNeoXModel(lowercase_					)
													original_model.to(lowercase_					)
													original_model.eval()
													SCREAMING_SNAKE_CASE       :   List[str]         =      original_model(lowercase_					).last_hidden_state
													SCREAMING_SNAKE_CASE       :   Dict         =      original_model(lowercase_					).last_hidden_state

													set_seed(42					)  # Fixed seed at init time so the two models get the same random weights
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      {'''type''': scaling_type, '''factor''': 10.0}
													SCREAMING_SNAKE_CASE       :   List[Any]         =      GPTNeoXModel(lowercase_					)
													scaled_model.to(lowercase_					)
													scaled_model.eval()
													SCREAMING_SNAKE_CASE       :   Tuple         =      scaled_model(lowercase_					).last_hidden_state
													SCREAMING_SNAKE_CASE       :   List[Any]         =      scaled_model(lowercase_					).last_hidden_state

													# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
													# maximum sequence length, so the outputs for the short input should match.
													if scaling_type == "dynamic":
																				self.assertTrue(torch.allclose(lowercase_					,     lowercase_					,     atol=1e-5					)					)
													else:
																				self.assertFalse(torch.allclose(lowercase_					,     lowercase_					,     atol=1e-5					)					)

													# The output should be different for long inputs
													self.assertFalse(torch.allclose(lowercase_					,     lowercase_					,     atol=1e-5					)					)


@require_torch
class 						lowerCamelCase_				(       unittest.TestCase      ):







						@slow
						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      AutoTokenizer.from_pretrained('''EleutherAI/pythia-410m-deduped'''					)
													for checkpointing in [True, False]:
																				SCREAMING_SNAKE_CASE       :   int         =      GPTNeoXForCausalLM.from_pretrained('''EleutherAI/pythia-410m-deduped'''					)

																				if checkpointing:
																											model.gradient_checkpointing_enable()
																				else:
																											model.gradient_checkpointing_disable()
																				model.to(lowercase_					)

																				SCREAMING_SNAKE_CASE       :   str         =      tokenizer('''My favorite food is'''					,     return_tensors='''pt'''					).to(lowercase_					)
																				# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
																				# See: https://github.com/huggingface/transformers/pull/24193
																				SCREAMING_SNAKE_CASE       :   Any         =      '''My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'''

																				SCREAMING_SNAKE_CASE       :   Optional[int]         =      model.generate(**lowercase_					,     do_sample=lowercase_					,     max_new_tokens=20					)
																				SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer.batch_decode(lowercase_					)[0]

																				self.assertEqual(lowercase_					,     lowercase_					)







"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCamelCase    : List[Any]				 =						logging.get_logger(__name__)

__UpperCamelCase    : Dict				 =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase    : List[Any]				 =						{
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCamelCase    : Union[str, Any]				 =						{
    '''distilbert-base-uncased''': 5_1_2,
    '''distilbert-base-uncased-distilled-squad''': 5_1_2,
    '''distilbert-base-cased''': 5_1_2,
    '''distilbert-base-cased-distilled-squad''': 5_1_2,
    '''distilbert-base-german-cased''': 5_1_2,
    '''distilbert-base-multilingual-cased''': 5_1_2,
}


__UpperCamelCase    : int				 =						{
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}







class 							SCREAMING_SNAKE_CASE   (						a_			):






					"""simple docstring"""



					lowercase__												=			VOCAB_FILES_NAMES
					lowercase__												=			PRETRAINED_VOCAB_FILES_MAP
					lowercase__												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__												=			PRETRAINED_INIT_CONFIGURATION
					lowercase__												=			["input_ids", "attention_mask"]
					lowercase__												=			DistilBertTokenizer







					def __init__(			self     :							int  ,lowercase_     :							List[str]=None  ,lowercase_     :							int=None  ,lowercase_     :							str=True  ,lowercase_     :							Union[str, Any]="[UNK]"  ,lowercase_     :							List[str]="[SEP]"  ,lowercase_     :							List[str]="[PAD]"  ,lowercase_     :							List[str]="[CLS]"  ,lowercase_     :							List[str]="[MASK]"  ,lowercase_     :							Tuple=True  ,lowercase_     :							Tuple=None  ,**lowercase_     :							str  ,):
						super().__init__(
						    lowercase_  ,tokenizer_file=lowercase_  ,do_lower_case=lowercase_  ,unk_token=lowercase_  ,sep_token=lowercase_  ,pad_token=lowercase_  ,cls_token=lowercase_  ,mask_token=lowercase_  ,tokenize_chinese_chars=lowercase_  ,strip_accents=lowercase_  ,**lowercase_  ,)

						lowerCAmelCase__      :		Tuple    							=      json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
						if (
						    normalizer_state.get('''lowercase'''  ,lowercase_						) != do_lower_case
						    or normalizer_state.get('''strip_accents'''  ,lowercase_						) != strip_accents
						    or normalizer_state.get('''handle_chinese_chars'''  ,lowercase_						) != tokenize_chinese_chars
						):
							lowerCAmelCase__      :		Union[str, Any]    							=      getattr(lowercase_  ,normalizer_state.pop('''type'''						)						)
							lowerCAmelCase__      :		Optional[int]    							=      do_lower_case
							lowerCAmelCase__      :		List[str]    							=      strip_accents
							lowerCAmelCase__      :		int    							=      tokenize_chinese_chars
							lowerCAmelCase__      :		Dict    							=      normalizer_class(**lowercase_						)

						lowerCAmelCase__      :		Tuple    							=      do_lower_case







					def 			__lowerCAmelCase					(			self     :							str  ,lowercase_     :							Tuple  ,lowercase_     :							Optional[Any]=None						):
						lowerCAmelCase__      :		Any    							=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

						return output







					def 			__lowerCAmelCase					(			self     :							int  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		Dict    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Dict    							=      [self.cls_token_id]
						if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
						return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							str  ,lowercase_     :							Optional[str] = None						):
						lowerCAmelCase__      :		Any    							=      self._tokenizer.model.save(lowercase_  ,name=lowercase_						)
						return tuple(lowercase_						)




