

'''simple docstring'''

import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    GPTNeoXForCausalLM,
							    GPTNeoXForQuestionAnswering,
							    GPTNeoXForSequenceClassification,
							    GPTNeoXForTokenClassification,
							    GPTNeoXModel,
							)






class    snake_case     :






				"""simple docstring"""



				def __init__(    self,     _lowercase,     _lowercase=13,     _lowercase=7,     _lowercase=True,     _lowercase=True,     _lowercase=True,     _lowercase=True,     _lowercase=99,     _lowercase=64,     _lowercase=5,     _lowercase=4,     _lowercase=37,     _lowercase="gelu",     _lowercase=0.1,     _lowercase=0.1,     _lowercase=512,     _lowercase=16,     _lowercase=2,     _lowercase=0.02,     _lowercase=3,     _lowercase=4,     _lowercase=None,     )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				parent
											SCREAMING_SNAKE_CASE_				=				batch_size
											SCREAMING_SNAKE_CASE_				=				seq_length
											SCREAMING_SNAKE_CASE_				=				is_training
											SCREAMING_SNAKE_CASE_				=				use_input_mask
											SCREAMING_SNAKE_CASE_				=				use_token_type_ids
											SCREAMING_SNAKE_CASE_				=				use_labels
											SCREAMING_SNAKE_CASE_				=				vocab_size
											SCREAMING_SNAKE_CASE_				=				hidden_size
											SCREAMING_SNAKE_CASE_				=				num_hidden_layers
											SCREAMING_SNAKE_CASE_				=				num_attention_heads
											SCREAMING_SNAKE_CASE_				=				intermediate_size
											SCREAMING_SNAKE_CASE_				=				hidden_act
											SCREAMING_SNAKE_CASE_				=				hidden_dropout_prob
											SCREAMING_SNAKE_CASE_				=				attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE_				=				max_position_embeddings
											SCREAMING_SNAKE_CASE_				=				type_vocab_size
											SCREAMING_SNAKE_CASE_				=				type_sequence_label_size
											SCREAMING_SNAKE_CASE_				=				initializer_range
											SCREAMING_SNAKE_CASE_				=				num_labels
											SCREAMING_SNAKE_CASE_				=				num_choices
											SCREAMING_SNAKE_CASE_				=				scope
											SCREAMING_SNAKE_CASE_				=				vocab_size - 1



				def   a__					(    self       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				ids_tensor([self.batch_size, self.seq_length],     self.vocab_size       )

											SCREAMING_SNAKE_CASE_				=				None
											if self.use_input_mask:
																		SCREAMING_SNAKE_CASE_				=				random_attention_mask([self.batch_size, self.seq_length]       )

											SCREAMING_SNAKE_CASE_				=				None
											if self.use_labels:
																		SCREAMING_SNAKE_CASE_				=				ids_tensor([self.batch_size, self.seq_length],     self.num_labels       )

											SCREAMING_SNAKE_CASE_				=				self.get_config()

											return config, input_ids, input_mask, token_labels



				def   a__					(    self       )     ->							Tuple:
											return GPTNeoXConfig(
											    vocab_size=self.vocab_size,     hidden_size=self.hidden_size,     num_hidden_layers=self.num_hidden_layers,     num_attention_heads=self.num_attention_heads,     intermediate_size=self.intermediate_size,     hidden_act=self.hidden_act,     hidden_dropout_prob=self.hidden_dropout_prob,     attention_probs_dropout_prob=self.attention_probs_dropout_prob,     max_position_embeddings=self.max_position_embeddings,     type_vocab_size=self.type_vocab_size,     is_decoder=__snake_case,     initializer_range=self.initializer_range,     pad_token_id=self.pad_token_id,     )



				def   a__					(    self       )     ->							Dict:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.prepare_config_and_inputs()

											SCREAMING_SNAKE_CASE_				=				True

											return config, input_ids, input_mask, token_labels



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase       )     ->							Optional[int]:
											SCREAMING_SNAKE_CASE_				=				GPTNeoXModel(config=__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case       )
											SCREAMING_SNAKE_CASE_				=				model(__snake_case       )
											self.parent.assertEqual(result.last_hidden_state.shape,     (self.batch_size, self.seq_length, self.hidden_size)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				True
											SCREAMING_SNAKE_CASE_				=				GPTNeoXModel(__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case       )
											self.parent.assertEqual(result.last_hidden_state.shape,     (self.batch_size, self.seq_length, self.hidden_size)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase,     _lowercase       )     ->							Dict:
											SCREAMING_SNAKE_CASE_				=				GPTNeoXForCausalLM(config=__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case,     labels=__snake_case       )
											self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.seq_length, self.vocab_size)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase,     _lowercase       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				self.num_labels
											SCREAMING_SNAKE_CASE_				=				GPTNeoXForQuestionAnswering(__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case       )
											self.parent.assertEqual(result.start_logits.shape,     (self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape,     (self.batch_size, self.seq_length)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase,     _lowercase       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				self.num_labels
											SCREAMING_SNAKE_CASE_				=				GPTNeoXForSequenceClassification(__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				ids_tensor([self.batch_size],     self.type_sequence_label_size       )
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case,     labels=__snake_case       )
											self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.num_labels)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase,     _lowercase       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				self.num_labels
											SCREAMING_SNAKE_CASE_				=				GPTNeoXForTokenClassification(__snake_case       )
											model.to(__snake_case       )
											model.eval()
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case,     labels=__snake_case       )
											self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.seq_length, self.num_labels)       )



				def   a__					(    self,     _lowercase,     _lowercase,     _lowercase       )     ->							List[str]:
											SCREAMING_SNAKE_CASE_				=				True
											SCREAMING_SNAKE_CASE_				=				GPTNeoXForCausalLM(config=__snake_case       )
											model.to(__snake_case       )
											model.eval()

											# first forward pass
											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case,     use_cache=__snake_case       )
											SCREAMING_SNAKE_CASE_				=				outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											SCREAMING_SNAKE_CASE_				=				ids_tensor((self.batch_size, 3),     config.vocab_size       )
											SCREAMING_SNAKE_CASE_				=				ids_tensor((self.batch_size, 3),     vocab_size=2       )

											# append to next input_ids and
											SCREAMING_SNAKE_CASE_				=				torch.cat([input_ids, next_tokens],     dim=-1       )
											SCREAMING_SNAKE_CASE_				=				torch.cat([input_mask, next_mask],     dim=-1       )

											SCREAMING_SNAKE_CASE_				=				model(__snake_case,     attention_mask=__snake_case,     output_hidden_states=__snake_case       )
											SCREAMING_SNAKE_CASE_				=				output_from_no_past['hidden_states'][0]
											SCREAMING_SNAKE_CASE_				=				model(
											    __snake_case,     attention_mask=__snake_case,     past_key_values=__snake_case,     output_hidden_states=__snake_case,     )['hidden_states'][0]

											# select random slice
											SCREAMING_SNAKE_CASE_				=				ids_tensor((1,),     output_from_past.shape[-1]       ).item()
											SCREAMING_SNAKE_CASE_				=				output_from_no_past[:, -3:, random_slice_idx].detach()
											SCREAMING_SNAKE_CASE_				=				output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(__snake_case,     __snake_case,     atol=1E-3       )       )





				def   a__					(    self       )     ->							int:
											SCREAMING_SNAKE_CASE_				=				self.prepare_config_and_inputs()
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				config_and_inputs
											SCREAMING_SNAKE_CASE_				=				{'input_ids': input_ids, 'attention_mask': input_mask}
											return config, inputs_dict






@require_torch
class    snake_case     (      _SCREAMING_SNAKE_CASE,						_SCREAMING_SNAKE_CASE,						_SCREAMING_SNAKE_CASE,						unittest.TestCase		):






				"""simple docstring"""
				_a    	=     (
				    (
				        GPTNeoXModel,
				        GPTNeoXForCausalLM,
				        GPTNeoXForQuestionAnswering,
				        GPTNeoXForSequenceClassification,
				        GPTNeoXForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				_a    	=     (GPTNeoXForCausalLM,) if is_torch_available() else ()
				_a    	=     (
				    {
				        """feature-extraction""": GPTNeoXModel,
				        """question-answering""": GPTNeoXForQuestionAnswering,
				        """text-classification""": GPTNeoXForSequenceClassification,
				        """text-generation""": GPTNeoXForCausalLM,
				        """token-classification""": GPTNeoXForTokenClassification,
				        """zero-shot""": GPTNeoXForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				_a    	=     False
				_a    	=     False
				_a    	=     False
				_a    	=     False



				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				GPTNeoXModelTester(self       )
											SCREAMING_SNAKE_CASE_				=				ConfigTester(self,     config_class=__snake_case,     hidden_size=64,     num_attention_heads=8       )



				def   a__					(    self       )     ->							int:
											self.config_tester.run_common_tests()



				def   a__					(    self       )     ->							List[Any]:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(__snake_case,     __snake_case,     __snake_case       )



				def   a__					(    self       )     ->							int:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(__snake_case,     __snake_case,     __snake_case       )



				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs_for_decoder()

											SCREAMING_SNAKE_CASE_				=				None

											self.model_tester.create_and_check_model_as_decoder(__snake_case,     __snake_case,     __snake_case       )



				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(__snake_case,     __snake_case,     __snake_case       )



				def   a__					(    self       )     ->							List[Any]:
											SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_causal_lm(*__snake_case       )



				def   a__					(    self       )     ->							Union[str, Any]:
											SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__snake_case       )



				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__snake_case       )



				def   a__					(    self       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__snake_case       )



				@unittest.skip(reason='Feed forward chunking is not implemented'       )
				def   a__					(    self       )     ->							str:
											pass





				@parameterized.expand([('linear',), ('dynamic',)]       )
				def   a__					(    self,     _lowercase       )     ->							str:
											SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				self.model_tester.prepare_config_and_inputs_for_common()
											SCREAMING_SNAKE_CASE_				=				ids_tensor([1, 10],     config.vocab_size       )
											SCREAMING_SNAKE_CASE_				=				ids_tensor([1, int(config.max_position_embeddings * 1.5       )],     config.vocab_size       )

											set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
											SCREAMING_SNAKE_CASE_				=				GPTNeoXModel(__snake_case       )
											original_model.to(__snake_case       )
											original_model.eval()
											SCREAMING_SNAKE_CASE_				=				original_model(__snake_case       ).last_hidden_state
											SCREAMING_SNAKE_CASE_				=				original_model(__snake_case       ).last_hidden_state

											set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
											SCREAMING_SNAKE_CASE_				=				{'type': scaling_type, 'factor': 10.0}
											SCREAMING_SNAKE_CASE_				=				GPTNeoXModel(__snake_case       )
											scaled_model.to(__snake_case       )
											scaled_model.eval()
											SCREAMING_SNAKE_CASE_				=				scaled_model(__snake_case       ).last_hidden_state
											SCREAMING_SNAKE_CASE_				=				scaled_model(__snake_case       ).last_hidden_state

											# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
											# maximum sequence length, so the outputs for the short input should match.
											if scaling_type == "dynamic":
																		self.assertTrue(torch.allclose(__snake_case,     __snake_case,     atol=1E-5       )       )
											else:
																		self.assertFalse(torch.allclose(__snake_case,     __snake_case,     atol=1E-5       )       )

											# The output should be different for long inputs
											self.assertFalse(torch.allclose(__snake_case,     __snake_case,     atol=1E-5       )       )






@require_torch
class    snake_case     (      unittest.TestCase		):






				"""simple docstring"""



				@slow
				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				AutoTokenizer.from_pretrained('EleutherAI/pythia-410m-deduped'       )
											for checkpointing in [True, False]:
																		SCREAMING_SNAKE_CASE_				=				GPTNeoXForCausalLM.from_pretrained('EleutherAI/pythia-410m-deduped'       )

																		if checkpointing:
																									model.gradient_checkpointing_enable()
																		else:
																									model.gradient_checkpointing_disable()
																		model.to(__snake_case       )

																		SCREAMING_SNAKE_CASE_				=				tokenizer('My favorite food is',     return_tensors='pt'       ).to(__snake_case       )
																		# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
																		# See: https://github.com/huggingface/transformers/pull/24193
																		SCREAMING_SNAKE_CASE_				=				'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

																		SCREAMING_SNAKE_CASE_				=				model.generate(**__snake_case,     do_sample=__snake_case,     max_new_tokens=20       )
																		SCREAMING_SNAKE_CASE_				=				tokenizer.batch_decode(__snake_case       )[0]

																		self.assertEqual(__snake_case,     __snake_case       )












import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class UpperCAmelCase_				( unittest.TestCase ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       0


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(GPTaTokenizer, GPTaTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,2_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoConfig.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,__snake_case )
       # Check that tokenizer_type ≠ model_type
       A_          =       AutoTokenizer.from_pretrained(__snake_case ,config=__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' )
            self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with pytest.raises(__snake_case ):
            AutoTokenizer.from_pretrained('''./''' ,tokenizer_type='''xxx''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            A_          =       tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )

            if isinstance(__snake_case ,__snake_case ):
                 self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,__snake_case )
            else:
                 self.assertEqual(tokenizer.do_lower_case ,__snake_case )

            self.assertEqual(tokenizer.model_max_length ,5_1_2 )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            with self.assertRaisesRegex(
                __snake_case ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier''' ,):
                 A_          =       tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       TOKENIZER_MAPPING.values()
       A_          =       []

       for slow_tok, fast_tok in tokenizers:
            if slow_tok is not None:
                 tokenizer_names.append(slow_tok.__name__ )

            if fast_tok is not None:
                 tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
            # must find the right class
            tokenizer_class_from_name(__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ,use_fast=__snake_case ) ,__snake_case )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ) ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''distilbert-base-uncased''' ,do_lower_case=__snake_case )
       A_          =       '''Hello, world. How are you?'''
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )

       A_          =       AutoTokenizer.from_pretrained('''microsoft/mpnet-base''' ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__snake_case ) ,__snake_case )
       self.assertEqual(tokenizer.model_max_length ,5_1_2 )
       self.assertEqual(tokenizer.vocab_size ,3_0_0_0_0 )
       self.assertEqual(tokenizer.unk_token ,'''[UNK]''' )
       self.assertEqual(tokenizer.padding_side ,'''right''' )
       self.assertEqual(tokenizer.truncation_side ,'''right''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case )

       self.assertIsInstance(__snake_case ,tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       get_tokenizer_config('''bert-base-cased''' )
       A_          =       config.pop('''_commit_hash''' ,__snake_case )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__snake_case ,{'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       A_          =       get_tokenizer_config(__snake_case )
       self.assertDictEqual(__snake_case ,{} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       get_tokenizer_config(__snake_case )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class'''] ,'''BertTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )

            A_          =       CustomTokenizer.from_pretrained(__snake_case )
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            # Can register in two steps
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None) )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            del TOKENIZER_MAPPING._extra_content[CustomConfig]
            # Can register in one step
            AutoTokenizer.register(
                __snake_case ,slow_tokenizer_class=__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )

            # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
            # and that model does not have a tokenizer.json
            with tempfile.TemporaryDirectory() as tmp_dir:
                 A_          =       BertTokenizerFast.from_pretrained(__snake_case )
                 bert_tokenizer.save_pretrained(__snake_case )
                 A_          =       CustomTokenizerFast.from_pretrained(__snake_case )

            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,use_fast=__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )

       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )
                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[int]							=   False





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[Any]							=   NewTokenizer
         __lowerCAmelCase   :						Union[str, Any]							=   False

       try:
            AutoConfig.register('''custom''' ,__snake_case )
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            # If remote code is not set, the default is to use local
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote code is disabled, we load the local one.
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote is enabled, we load from the Hub
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertTrue(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(tokenizer.special_attribute_present )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,'''bert-base is not a local folder and is not a valid model identifier''' ):
            A_          =       AutoTokenizer.from_pretrained('''bert-base''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,revision='''aaaaaa''' )




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
            self.assertEqual(counter.get_request_count ,0 )
            self.assertEqual(counter.head_request_count ,1 )
            self.assertEqual(counter.other_request_count ,0 )



