





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    AlbertForMaskedLM,
		    AlbertForMultipleChoice,
		    AlbertForPreTraining,
		    AlbertForQuestionAnswering,
		    AlbertForSequenceClassification,
		    AlbertForTokenClassification,
		    AlbertModel,
		)
		from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 						UpperCamelCase_  :



					def __init__(		self     :     Optional[Any]       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     str=13       ,      lowerCamelCase     :     Dict=7       ,      lowerCamelCase     :     int=True       ,      lowerCamelCase     :     Tuple=True       ,      lowerCamelCase     :     List[str]=True       ,      lowerCamelCase     :     Any=True       ,      lowerCamelCase     :     Union[str, Any]=99       ,      lowerCamelCase     :     Tuple=16       ,      lowerCamelCase     :     Dict=36       ,      lowerCamelCase     :     List[str]=6       ,      lowerCamelCase     :     List[str]=6       ,      lowerCamelCase     :     Optional[int]=6       ,      lowerCamelCase     :     List[str]=37       ,      lowerCamelCase     :     Dict="gelu"       ,      lowerCamelCase     :     Any=0.1       ,      lowerCamelCase     :     List[str]=0.1       ,      lowerCamelCase     :     List[Any]=5_12       ,      lowerCamelCase     :     Union[str, Any]=16       ,      lowerCamelCase     :     Union[str, Any]=2       ,      lowerCamelCase     :     Dict=0.02       ,      lowerCamelCase     :     Dict=3       ,      lowerCamelCase     :     Union[str, Any]=4       ,      lowerCamelCase     :     Dict=None       ,      ):
							lowerCamelCase_	:     Optional[Any] 						=      parent
							lowerCamelCase_	:     List[Any] 						=      batch_size
							lowerCamelCase_	:     Any 						=      seq_length
							lowerCamelCase_	:     Any 						=      is_training
							lowerCamelCase_	:     Dict 						=      use_input_mask
							lowerCamelCase_	:     int 						=      use_token_type_ids
							lowerCamelCase_	:     Dict 						=      use_labels
							lowerCamelCase_	:     str 						=      vocab_size
							lowerCamelCase_	:     Tuple 						=      embedding_size
							lowerCamelCase_	:     str 						=      hidden_size
							lowerCamelCase_	:     Dict 						=      num_hidden_layers
							lowerCamelCase_	:     str 						=      num_hidden_groups
							lowerCamelCase_	:     List[str] 						=      num_attention_heads
							lowerCamelCase_	:     List[Any] 						=      intermediate_size
							lowerCamelCase_	:     List[str] 						=      hidden_act
							lowerCamelCase_	:     Optional[int] 						=      hidden_dropout_prob
							lowerCamelCase_	:     Any 						=      attention_probs_dropout_prob
							lowerCamelCase_	:     List[str] 						=      max_position_embeddings
							lowerCamelCase_	:     Any 						=      type_vocab_size
							lowerCamelCase_	:     int 						=      type_sequence_label_size
							lowerCamelCase_	:     Optional[int] 						=      initializer_range
							lowerCamelCase_	:     str 						=      num_labels
							lowerCamelCase_	:     Tuple 						=      num_choices
							lowerCamelCase_	:     str 						=      scope



					def   __a     (		self     :     Any						):
							lowerCamelCase_	:     str 						=      ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size						)

							lowerCamelCase_	:     str 						=      None
							if self.use_input_mask:
									lowerCamelCase_	:     Union[str, Any] 						=      random_attention_mask([self.batch_size, self.seq_length]						)

							lowerCamelCase_	:     str 						=      None
							if self.use_token_type_ids:
									lowerCamelCase_	:     int 						=      ids_tensor([self.batch_size, self.seq_length]       ,      self.type_vocab_size						)

							lowerCamelCase_	:     Any 						=      None
							lowerCamelCase_	:     List[str] 						=      None
							lowerCamelCase_	:     Optional[Any] 						=      None
							if self.use_labels:
									lowerCamelCase_	:     str 						=      ids_tensor([self.batch_size]       ,      self.type_sequence_label_size						)
									lowerCamelCase_	:     List[str] 						=      ids_tensor([self.batch_size, self.seq_length]       ,      self.num_labels						)
									lowerCamelCase_	:     Dict 						=      ids_tensor([self.batch_size]       ,      self.num_choices						)

							lowerCamelCase_	:     Dict 						=      self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



					def   __a     (		self     :     Any						):
							return AlbertConfig(
							    vocab_size=self.vocab_size       ,      hidden_size=self.hidden_size       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      type_vocab_size=self.type_vocab_size       ,      initializer_range=self.initializer_range       ,      num_hidden_groups=self.num_hidden_groups       ,      )



					def   __a     (		self     :     List[str]       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     Any       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Tuple       ,      lowerCamelCase     :     Dict       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     Optional[int]						):
							lowerCamelCase_	:     Tuple 						=      AlbertModel(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     int 						=      model(_lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase						)
							lowerCamelCase_	:     Any 						=      model(_lowerCamelCase       ,      token_type_ids=_lowerCamelCase						)
							lowerCamelCase_	:     Union[str, Any] 						=      model(_lowerCamelCase						)
							self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)						)
							self.parent.assertEqual(result.pooler_output.shape       ,      (self.batch_size, self.hidden_size)						)



					def   __a     (		self     :     Optional[Any]       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     int       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Tuple       ,      lowerCamelCase     :     Optional[int]       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Union[str, Any]						):
							lowerCamelCase_	:     Any 						=      AlbertForPreTraining(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     Any 						=      model(
							    _lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      labels=_lowerCamelCase       ,      sentence_order_label=_lowerCamelCase       ,      )
							self.parent.assertEqual(result.prediction_logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)						)
							self.parent.assertEqual(result.sop_logits.shape       ,      (self.batch_size, config.num_labels)						)



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Tuple       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Any       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     Union[str, Any]						):
							lowerCamelCase_	:     Dict 						=      AlbertForMaskedLM(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     List[str] 						=      model(_lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      labels=_lowerCamelCase						)
							self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)						)



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Tuple       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     Optional[int]       ,      lowerCamelCase     :     Any						):
							lowerCamelCase_	:     int 						=      AlbertForQuestionAnswering(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     Union[str, Any] 						=      model(
							    _lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      start_positions=_lowerCamelCase       ,      end_positions=_lowerCamelCase       ,      )
							self.parent.assertEqual(result.start_logits.shape       ,      (self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape       ,      (self.batch_size, self.seq_length)						)



					def   __a     (		self     :     Optional[Any]       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Dict       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     int       ,      lowerCamelCase     :     int       ,      lowerCamelCase     :     Tuple						):
							lowerCamelCase_	:     Optional[int] 						=      self.num_labels
							lowerCamelCase_	:     int 						=      AlbertForSequenceClassification(_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     Optional[Any] 						=      model(_lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      labels=_lowerCamelCase						)
							self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_labels)						)



					def   __a     (		self     :     Optional[Any]       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Tuple       ,      lowerCamelCase     :     List[str]       ,      lowerCamelCase     :     Any       ,      lowerCamelCase     :     Optional[int]       ,      lowerCamelCase     :     List[str]       ,      lowerCamelCase     :     List[Any]						):
							lowerCamelCase_	:     int 						=      self.num_labels
							lowerCamelCase_	:     Any 						=      AlbertForTokenClassification(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     Any 						=      model(_lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      labels=_lowerCamelCase						)
							self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.num_labels)						)



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     int       ,      lowerCamelCase     :     List[str]       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     List[Any]       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Dict						):
							lowerCamelCase_	:     str 						=      self.num_choices
							lowerCamelCase_	:     Tuple 						=      AlbertForMultipleChoice(config=_lowerCamelCase						)
							model.to(_lowerCamelCase						)
							model.eval()
							lowerCamelCase_	:     List[str] 						=      input_ids.unsqueeze(1						).expand(-1       ,      self.num_choices       ,      -1						).contiguous()
							lowerCamelCase_	:     Optional[Any] 						=      token_type_ids.unsqueeze(1						).expand(-1       ,      self.num_choices       ,      -1						).contiguous()
							lowerCamelCase_	:     Tuple 						=      input_mask.unsqueeze(1						).expand(-1       ,      self.num_choices       ,      -1						).contiguous()
							lowerCamelCase_	:     List[str] 						=      model(
							    _lowerCamelCase       ,      attention_mask=_lowerCamelCase       ,      token_type_ids=_lowerCamelCase       ,      labels=_lowerCamelCase       ,      )
							self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_choices)						)





					def   __a     (		self     :     int						):
							lowerCamelCase_	:     Union[str, Any] 						=      self.prepare_config_and_inputs()
							(
							    (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      (
							    lowerCamelCase_					
							)						,      					
							)	:     Any 						=      config_and_inputs
							lowerCamelCase_	:     Union[str, Any] 						=      {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict






@require_torch
class 						UpperCamelCase_  (  __a  ,     __a  ,     unittest.TestCase							):
					_a      :						Optional[int]						    =					(
					    (
					        AlbertModel,
					        AlbertForPreTraining,
					        AlbertForMaskedLM,
					        AlbertForMultipleChoice,
					        AlbertForSequenceClassification,
					        AlbertForTokenClassification,
					        AlbertForQuestionAnswering,
					    )
					    if is_torch_available()
					    else ()
					)
					_a      :						Optional[int]						    =					(
					    {
					        """feature-extraction""": AlbertModel,
					        """fill-mask""": AlbertForMaskedLM,
					        """question-answering""": AlbertForQuestionAnswering,
					        """text-classification""": AlbertForSequenceClassification,
					        """token-classification""": AlbertForTokenClassification,
					        """zero-shot""": AlbertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					_a      :						Optional[Any]						    =					True



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Any       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Tuple=False						):
							lowerCamelCase_	:     int 						=      super()._prepare_for_class(_lowerCamelCase       ,      _lowerCamelCase       ,      return_labels=_lowerCamelCase						)

							if return_labels:
									if model_class in get_values(_lowerCamelCase						):
											lowerCamelCase_	:     Optional[Any] 						=      torch.zeros(
											    (self.model_tester.batch_size, self.model_tester.seq_length)       ,      dtype=torch.long       ,      device=_lowerCamelCase						)
											lowerCamelCase_	:     str 						=      torch.zeros(
											    self.model_tester.batch_size       ,      dtype=torch.long       ,      device=_lowerCamelCase						)
							return inputs_dict



					def   __a     (		self     :     Dict						):
							lowerCamelCase_	:     List[str] 						=      AlbertModelTester(self						)
							lowerCamelCase_	:     str 						=      ConfigTester(self       ,      config_class=_lowerCamelCase       ,      hidden_size=37						)



					def   __a     (		self     :     str						):
							self.config_tester.run_common_tests()



					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     List[Any] 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_lowerCamelCase						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     Dict 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     Tuple 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase						)



					def   __a     (		self     :     Any						):
							lowerCamelCase_	:     Union[str, Any] 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase						)



					def   __a     (		self     :     int						):
							lowerCamelCase_	:     List[Any] 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase						)



					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     str 						=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase						)



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     List[Any] 						=      self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
									lowerCamelCase_	:     List[str] 						=      type
									self.model_tester.create_and_check_model(*_lowerCamelCase						)





					@slow
					def   __a     (		self     :     Dict						):
							for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_	:     List[str] 						=      AlbertModel.from_pretrained(_lowerCamelCase						)
									self.assertIsNotNone(_lowerCamelCase						)






@require_torch
class 						UpperCamelCase_  (  unittest.TestCase							):



					@slow
					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     List[str] 						=      AlbertModel.from_pretrained('albert-base-v2'						)
							lowerCamelCase_	:     int 						=      torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]						)
							lowerCamelCase_	:     Optional[int] 						=      torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]						)
							with torch.no_grad():
									lowerCamelCase_	:     Union[str, Any] 						=      model(_lowerCamelCase       ,      attention_mask=_lowerCamelCase						)[0]
							lowerCamelCase_	:     str 						=      torch.Size((1, 11, 7_68)						)
							self.assertEqual(output.shape       ,      _lowerCamelCase						)
							lowerCamelCase_	:     Dict 						=      torch.tensor(
							    [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]						)

							self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,      _lowerCamelCase       ,      atol=1E-4						)						)


"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast

class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										try:
													__lowercase       =       tempfile.mktemp()
													with open(_lowerCamelCase  ,    "wb"       ) as f:
																http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  ,    _lowerCamelCase       )

													__lowercase       =       AlbertTokenizer.from_pretrained(_lowerCamelCase       )
										finally:
													os.remove(_lowerCamelCase       )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("tokenizer.json"       ):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
										try:
													with open("tokenizer.json"  ,    "wb"       ) as f:
																http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"  ,    _lowerCamelCase       )
													__lowercase       =       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size  ,    1_000       )
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
													os.remove("tokenizer.json"       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										__lowercase       =       AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )





@is_staging_test
class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							_lowerCamelCase					:			str          =   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[Any]:




										'''simple docstring'''
										__lowercase       =       TOKEN
										HfFolder.save_token(_lowerCamelCase       )
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[str]:




										'''simple docstring'''
										try:
													delete_repo(token=cls._token  ,    repo_id="test-tokenizer"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="valid_org/test-tokenizer-org"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="test-dynamic-tokenizer"       )
										except HTTPError:
													pass
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("test-tokenizer"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="test-tokenizer"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_lowerCamelCase  ,    repo_id="test-tokenizer"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("valid_org/test-tokenizer-org"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="valid_org/test-tokenizer-org"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    _lowerCamelCase  ,    repo_id="valid_org/test-tokenizer-org"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )





							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       CustomTokenizer(_lowerCamelCase       )

										# No fast custom tokenizer
										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

													__lowercase       =       BertTokenizerFast.from_pretrained(_lowerCamelCase       )
													bert_tokenizer.save_pretrained(_lowerCamelCase       )
													__lowercase       =       CustomTokenizerFast.from_pretrained(_lowerCamelCase       )

										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizerFast"       )
										__lowercase       =       AutoTokenizer.from_pretrained(
										    f'''{USER}/test-dynamic-tokenizer'''  ,    use_fast=_lowerCamelCase  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )





class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("Hello 友達"       )
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
										trie.add("Hello"       )
										trie.data
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS] This is a extra_id_100"]       )
										trie.add("[CLS]"       )
										trie.add("extra_id_1"       )
										trie.add("extra_id_100"       )
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS]", " This is a ", "extra_id_100"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["A", "BC"]       )
										self.assertEqual(trie.split("BCA"       )  ,    ["BC", "A"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("TOKEN]"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										trie.add("P"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("AB"       )
										trie.add("B"       )
										trie.add("C"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["AB", "C"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("ABC"       )
										trie.add("B"       )
										trie.add("CD"       )
										self.assertEqual(trie.split("ABCD"       )  ,    ["ABC", "D"]       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# Even if the offsets are wrong, we necessarily output correct string
										# parts.
										__lowercase       =       Trie()
										__lowercase       =       trie.cut_text("ABC"  ,    [0, 0, 2, 1, 2, 3]       )
										self.assertEqual(_lowerCamelCase  ,    ["AB", "C"]       )





