
"""simple docstring"""






import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    GPTNeoXForCausalLM,
					    GPTNeoXForQuestionAnswering,
					    GPTNeoXForSequenceClassification,
					    GPTNeoXForTokenClassification,
					    GPTNeoXModel,
					)

class 					__a     :



							'''simple docstring'''







							def __init__(     self  ,    _lowerCamelCase  ,    _lowerCamelCase=13  ,    _lowerCamelCase=7  ,    _lowerCamelCase=True  ,    _lowerCamelCase=True  ,    _lowerCamelCase=True  ,    _lowerCamelCase=True  ,    _lowerCamelCase=99  ,    _lowerCamelCase=64  ,    _lowerCamelCase=5  ,    _lowerCamelCase=4  ,    _lowerCamelCase=37  ,    _lowerCamelCase="gelu"  ,    _lowerCamelCase=0.1  ,    _lowerCamelCase=0.1  ,    _lowerCamelCase=512  ,    _lowerCamelCase=16  ,    _lowerCamelCase=2  ,    _lowerCamelCase=0.02  ,    _lowerCamelCase=3  ,    _lowerCamelCase=4  ,    _lowerCamelCase=None  ,    )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       parent
										__lowercase       =       batch_size
										__lowercase       =       seq_length
										__lowercase       =       is_training
										__lowercase       =       use_input_mask
										__lowercase       =       use_token_type_ids
										__lowercase       =       use_labels
										__lowercase       =       vocab_size
										__lowercase       =       hidden_size
										__lowercase       =       num_hidden_layers
										__lowercase       =       num_attention_heads
										__lowercase       =       intermediate_size
										__lowercase       =       hidden_act
										__lowercase       =       hidden_dropout_prob
										__lowercase       =       attention_probs_dropout_prob
										__lowercase       =       max_position_embeddings
										__lowercase       =       type_vocab_size
										__lowercase       =       type_sequence_label_size
										__lowercase       =       initializer_range
										__lowercase       =       num_labels
										__lowercase       =       num_choices
										__lowercase       =       scope
										__lowercase       =       vocab_size - 1
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size       )

										__lowercase       =       None
										if self.use_input_mask:
													__lowercase       =       random_attention_mask([self.batch_size, self.seq_length]       )

										__lowercase       =       None
										if self.use_labels:
													__lowercase       =       ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels       )

										__lowercase       =       self.get_config()

										return config, input_ids, input_mask, token_labels
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return GPTNeoXConfig(
										    vocab_size=self.vocab_size  ,    hidden_size=self.hidden_size  ,    num_hidden_layers=self.num_hidden_layers  ,    num_attention_heads=self.num_attention_heads  ,    intermediate_size=self.intermediate_size  ,    hidden_act=self.hidden_act  ,    hidden_dropout_prob=self.hidden_dropout_prob  ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,    max_position_embeddings=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    is_decoder=_lowerCamelCase  ,    initializer_range=self.initializer_range  ,    pad_token_id=self.pad_token_id  ,    )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       self.prepare_config_and_inputs()

										__lowercase       =       True

										return config, input_ids, input_mask, token_labels
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							str:




										'''simple docstring'''
										__lowercase       =       GPTNeoXModel(config=_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase       )
										__lowercase       =       model(_lowerCamelCase       )
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.hidden_size)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       True
										__lowercase       =       GPTNeoXModel(_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase       )
										self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.hidden_size)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							List[str]:




										'''simple docstring'''
										__lowercase       =       GPTNeoXForCausalLM(config=_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    labels=_lowerCamelCase       )
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       self.num_labels
										__lowercase       =       GPTNeoXForQuestionAnswering(_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase       )
										self.parent.assertEqual(result.start_logits.shape  ,    (self.batch_size, self.seq_length)       )
										self.parent.assertEqual(result.end_logits.shape  ,    (self.batch_size, self.seq_length)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							int:




										'''simple docstring'''
										__lowercase       =       self.num_labels
										__lowercase       =       GPTNeoXForSequenceClassification(_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       ids_tensor([self.batch_size]  ,    self.type_sequence_label_size       )
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    labels=_lowerCamelCase       )
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_labels)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							List[str]:




										'''simple docstring'''
										__lowercase       =       self.num_labels
										__lowercase       =       GPTNeoXForTokenClassification(_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    labels=_lowerCamelCase       )
										self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.num_labels)       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							Any:




										'''simple docstring'''
										__lowercase       =       True
										__lowercase       =       GPTNeoXForCausalLM(config=_lowerCamelCase       )
										model.to(_lowerCamelCase       )
										model.eval()

										# first forward pass
										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    use_cache=_lowerCamelCase       )
										__lowercase       =       outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										__lowercase       =       ids_tensor((self.batch_size, 3)  ,    config.vocab_size       )
										__lowercase       =       ids_tensor((self.batch_size, 3)  ,    vocab_size=2       )

										# append to next input_ids and
										__lowercase       =       torch.cat([input_ids, next_tokens]  ,    dim=-1       )
										__lowercase       =       torch.cat([input_mask, next_mask]  ,    dim=-1       )

										__lowercase       =       model(_lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    output_hidden_states=_lowerCamelCase       )
										__lowercase       =       output_from_no_past["hidden_states"][0]
										__lowercase       =       model(
										    _lowerCamelCase  ,    attention_mask=_lowerCamelCase  ,    past_key_values=_lowerCamelCase  ,    output_hidden_states=_lowerCamelCase  ,    )["hidden_states"][0]

										# select random slice
										__lowercase       =       ids_tensor((1,)  ,    output_from_past.shape[-1]       ).item()
										__lowercase       =       output_from_no_past[:, -3:, random_slice_idx].detach()
										__lowercase       =       output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(_lowerCamelCase  ,    _lowerCamelCase  ,    atol=1e-3       )       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       self.prepare_config_and_inputs()
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       config_and_inputs
										__lowercase       =       {"input_ids": input_ids, "attention_mask": input_mask}
										return config, inputs_dict





@require_torch
class 					__a     ( __a					,   __a					,   __a					,   unittest.TestCase       ):



							'''simple docstring'''







							_lowerCamelCase					:			Any          =   (
							    (
							        GPTNeoXModel,
							        GPTNeoXForCausalLM,
							        GPTNeoXForQuestionAnswering,
							        GPTNeoXForSequenceClassification,
							        GPTNeoXForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							_lowerCamelCase					:			Union[str, Any]          =   (GPTNeoXForCausalLM,) if is_torch_available() else ()
							_lowerCamelCase					:			Tuple          =   (
							    {
							        """feature-extraction""": GPTNeoXModel,
							        """question-answering""": GPTNeoXForQuestionAnswering,
							        """text-classification""": GPTNeoXForSequenceClassification,
							        """text-generation""": GPTNeoXForCausalLM,
							        """token-classification""": GPTNeoXForTokenClassification,
							        """zero-shot""": GPTNeoXForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_lowerCamelCase					:			str          =   False
							_lowerCamelCase					:			Tuple          =   False
							_lowerCamelCase					:			str          =   False
							_lowerCamelCase					:			List[Any]          =   False
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       GPTNeoXModelTester(self       )
										__lowercase       =       ConfigTester(self  ,    config_class=_lowerCamelCase  ,    hidden_size=64  ,    num_attention_heads=8       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										self.config_tester.run_common_tests()
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# This regression test was failing with PyTorch < 1.3
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       self.model_tester.prepare_config_and_inputs_for_decoder()

										__lowercase       =       None

										self.model_tester.create_and_check_model_as_decoder(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase   ,					__lowercase   ,					__lowercase   ,					__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_decoder_model_past_large_inputs(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_causal_lm(*_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[str]:




										'''simple docstring'''
										__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										__lowercase       =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase       )
							@unittest.skip(reason="Feed forward chunking is not implemented"       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										pass





							@parameterized.expand([("linear",), ("dynamic",)]       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase   ,					__lowercase       =       self.model_tester.prepare_config_and_inputs_for_common()
										__lowercase       =       ids_tensor([1, 10]  ,    config.vocab_size       )
										__lowercase       =       ids_tensor([1, int(config.max_position_embeddings * 1.5       )]  ,    config.vocab_size       )

										set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
										__lowercase       =       GPTNeoXModel(_lowerCamelCase       )
										original_model.to(_lowerCamelCase       )
										original_model.eval()
										__lowercase       =       original_model(_lowerCamelCase       ).last_hidden_state
										__lowercase       =       original_model(_lowerCamelCase       ).last_hidden_state

										set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
										__lowercase       =       {"type": scaling_type, "factor": 10.0}
										__lowercase       =       GPTNeoXModel(_lowerCamelCase       )
										scaled_model.to(_lowerCamelCase       )
										scaled_model.eval()
										__lowercase       =       scaled_model(_lowerCamelCase       ).last_hidden_state
										__lowercase       =       scaled_model(_lowerCamelCase       ).last_hidden_state

										# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
										# maximum sequence length, so the outputs for the short input should match.
										if scaling_type == "dynamic":
													self.assertTrue(torch.allclose(_lowerCamelCase  ,    _lowerCamelCase  ,    atol=1e-5       )       )
										else:
													self.assertFalse(torch.allclose(_lowerCamelCase  ,    _lowerCamelCase  ,    atol=1e-5       )       )

										# The output should be different for long inputs
										self.assertFalse(torch.allclose(_lowerCamelCase  ,    _lowerCamelCase  ,    atol=1e-5       )       )





@require_torch
class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							@slow
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										__lowercase       =       AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped"       )
										for checkpointing in [True, False]:
													__lowercase       =       GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped"       )

													if checkpointing:
																model.gradient_checkpointing_enable()
													else:
																model.gradient_checkpointing_disable()
													model.to(_lowerCamelCase       )

													__lowercase       =       tokenizer("My favorite food is"  ,    return_tensors="pt"       ).to(_lowerCamelCase       )
													# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
													# See: https://github.com/huggingface/transformers/pull/24193
													__lowercase       =       "My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"

													__lowercase       =       model.generate(**_lowerCamelCase  ,    do_sample=_lowerCamelCase  ,    max_new_tokens=20       )
													__lowercase       =       tokenizer.batch_decode(_lowerCamelCase       )[0]

													self.assertEqual(_lowerCamelCase  ,    _lowerCamelCase       )







"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast

class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										try:
													__lowercase       =       tempfile.mktemp()
													with open(_lowerCamelCase  ,    "wb"       ) as f:
																http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  ,    _lowerCamelCase       )

													__lowercase       =       AlbertTokenizer.from_pretrained(_lowerCamelCase       )
										finally:
													os.remove(_lowerCamelCase       )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("tokenizer.json"       ):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
										try:
													with open("tokenizer.json"  ,    "wb"       ) as f:
																http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"  ,    _lowerCamelCase       )
													__lowercase       =       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size  ,    1_000       )
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
													os.remove("tokenizer.json"       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										__lowercase       =       AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )





@is_staging_test
class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							_lowerCamelCase					:			str          =   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[Any]:




										'''simple docstring'''
										__lowercase       =       TOKEN
										HfFolder.save_token(_lowerCamelCase       )
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[str]:




										'''simple docstring'''
										try:
													delete_repo(token=cls._token  ,    repo_id="test-tokenizer"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="valid_org/test-tokenizer-org"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="test-dynamic-tokenizer"       )
										except HTTPError:
													pass
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("test-tokenizer"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="test-tokenizer"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_lowerCamelCase  ,    repo_id="test-tokenizer"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("valid_org/test-tokenizer-org"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="valid_org/test-tokenizer-org"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    _lowerCamelCase  ,    repo_id="valid_org/test-tokenizer-org"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )





							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       CustomTokenizer(_lowerCamelCase       )

										# No fast custom tokenizer
										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

													__lowercase       =       BertTokenizerFast.from_pretrained(_lowerCamelCase       )
													bert_tokenizer.save_pretrained(_lowerCamelCase       )
													__lowercase       =       CustomTokenizerFast.from_pretrained(_lowerCamelCase       )

										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizerFast"       )
										__lowercase       =       AutoTokenizer.from_pretrained(
										    f'''{USER}/test-dynamic-tokenizer'''  ,    use_fast=_lowerCamelCase  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )





class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("Hello 友達"       )
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
										trie.add("Hello"       )
										trie.data
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS] This is a extra_id_100"]       )
										trie.add("[CLS]"       )
										trie.add("extra_id_1"       )
										trie.add("extra_id_100"       )
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS]", " This is a ", "extra_id_100"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["A", "BC"]       )
										self.assertEqual(trie.split("BCA"       )  ,    ["BC", "A"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("TOKEN]"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										trie.add("P"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("AB"       )
										trie.add("B"       )
										trie.add("C"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["AB", "C"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("ABC"       )
										trie.add("B"       )
										trie.add("CD"       )
										self.assertEqual(trie.split("ABCD"       )  ,    ["ABC", "D"]       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# Even if the offsets are wrong, we necessarily output correct string
										# parts.
										__lowercase       =       Trie()
										__lowercase       =       trie.cut_text("ABC"  ,    [0, 0, 2, 1, 2, 3]       )
										self.assertEqual(_lowerCamelCase  ,    ["AB", "C"]       )





