


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class      __lowercase  (__SCREAMING_SNAKE_CASE     ,    unittest.TestCase   ):





							"""simple docstring"""







							_UpperCAmelCase     =		PreTrainedTokenizerFast
							_UpperCAmelCase     =		False
							_UpperCAmelCase     =		True
							_UpperCAmelCase     =		"""tokenizer_file"""






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Any       						= False  # because we don't have pretrained_vocab_files_map
										super().setUp()
										SCREAMING_SNAKE_CASE_						:			Tuple       						= True

										SCREAMING_SNAKE_CASE_						:			List[str]       						= ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= 'SaulLu/dummy-tokenizer-bytelevel-bpe'

										# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

										SCREAMING_SNAKE_CASE_						:			Dict       						= PreTrainedTokenizerFast.from_pretrained(model_paths[0]    )
										tokenizer.save_pretrained(self.tmpdirname    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										pass






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										pass






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										pass






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										pass






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																try:
																			SCREAMING_SNAKE_CASE_						:			List[str]       						= tempfile.mkdtemp()
																			SCREAMING_SNAKE_CASE_						:			str       						= self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__       ,		**lowerCAmelCase__    )

																			tokenizer.save_pretrained(self.tmpdirname    )
																			super().test_training_new_tokenizer()
																finally:
																			# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																			# is restored
																			shutil.rmtree(self.tmpdirname    )
																			SCREAMING_SNAKE_CASE_						:			Dict       						= tmpdirname_orig






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Any       						= self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																try:
																			SCREAMING_SNAKE_CASE_						:			Optional[int]       						= tempfile.mkdtemp()
																			SCREAMING_SNAKE_CASE_						:			Dict       						= self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__       ,		**lowerCAmelCase__    )

																			tokenizer.save_pretrained(self.tmpdirname    )
																			super().test_training_new_tokenizer_with_special_tokens_change()
																finally:
																			# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																			# is restored
																			shutil.rmtree(self.tmpdirname    )
																			SCREAMING_SNAKE_CASE_						:			Any       						= tmpdirname_orig






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Any       						= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name    )

										SCREAMING_SNAKE_CASE_						:			List[str]       						= ('a' for _ in range(1_0_0_0    ))
										SCREAMING_SNAKE_CASE_						:			List[str]       						= tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__       ,		length=1_0_0_0       ,		vocab_size=5_0    )

										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= new_tokenizer.encode('a🤗'    )
										self.assertEqual(lowerCAmelCase__       ,		[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]    )







@require_tokenizers
class      __lowercase  (unittest.TestCase   ):





							"""simple docstring"""







							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Any       						= AutoTokenizer.from_pretrained('bert-base-cased'    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= json.loads(tokenizer._tokenizer.to_str()    )
										SCREAMING_SNAKE_CASE_						:			List[str]       						= len(lowerCAmelCase__    )

										with tempfile.TemporaryDirectory() as tmp_dir:
													# Hack to save this in the tokenizer_config.json
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= ['tokenizer.4.0.0.json']
													tokenizer.save_pretrained(lowerCAmelCase__    )
													json.dump(lowerCAmelCase__       ,		open(os.path.join(lowerCAmelCase__       ,		'tokenizer.4.0.0.json'    )       ,		'w'    )    )

													# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
													SCREAMING_SNAKE_CASE_						:			str       						= AutoTokenizer.from_pretrained(lowerCAmelCase__    )
													self.assertEqual(len(lowerCAmelCase__    )       ,		len(lowerCAmelCase__    ) + 1    )
													SCREAMING_SNAKE_CASE_						:			str       						= json.loads(new_tokenizer._tokenizer.to_str()    )
													self.assertIn('huggingface'       ,		json_tokenizer['model']['vocab']    )

													# Will need to be adjusted if we reach v42 and this test is still here.
													# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
													shutil.move(os.path.join(lowerCAmelCase__       ,		'tokenizer.4.0.0.json'    )       ,		os.path.join(lowerCAmelCase__       ,		'tokenizer.42.0.0.json'    )    )
													SCREAMING_SNAKE_CASE_						:			int       						= ['tokenizer.42.0.0.json']
													tokenizer.save_pretrained(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			List[str]       						= AutoTokenizer.from_pretrained(lowerCAmelCase__    )
													self.assertEqual(len(lowerCAmelCase__    )       ,		len(lowerCAmelCase__    )    )
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= json.loads(new_tokenizer._tokenizer.to_str()    )
													self.assertNotIn('huggingface'       ,		json_tokenizer['model']['vocab']    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= 'hf-internal-testing/test-two-tokenizers'

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= AutoTokenizer.from_pretrained(lowerCAmelCase__    )
										self.assertEqual(len(lowerCAmelCase__    )       ,		2_8_9_9_7    )
										SCREAMING_SNAKE_CASE_						:			Tuple       						= json.loads(tokenizer._tokenizer.to_str()    )
										self.assertIn('huggingface'       ,		json_tokenizer['model']['vocab']    )

										# Testing an older version by monkey-patching the version in the module it's used.
										import transformers as old_transformers

										SCREAMING_SNAKE_CASE_						:			List[str]       						= '3.0.0'
										SCREAMING_SNAKE_CASE_						:			str       						= old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__    )
										self.assertEqual(len(lowerCAmelCase__    )       ,		2_8_9_9_6    )
										SCREAMING_SNAKE_CASE_						:			Tuple       						= json.loads(old_tokenizer._tokenizer.to_str()    )
										self.assertNotIn('huggingface'       ,		json_tokenizer['model']['vocab']    )







@require_tokenizers
class      __lowercase  (unittest.TestCase   ):





							"""simple docstring"""







							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Tuple       						= PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'    )
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= 'The Matrix is a 1999 science fiction action film.'

										with concurrent.futures.ThreadPoolExecutor() as executor:
													SCREAMING_SNAKE_CASE_						:			Tuple       						= [executor.submit(self.fetch       ,		lowerCAmelCase__       ,		lowerCAmelCase__    ) for i in range(1_0    )]
													SCREAMING_SNAKE_CASE_						:			Dict       						= [future.result() for future in futures]
													self.assertEqual(lowerCAmelCase__       ,		[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0    )]    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										return tokenizer.encode(lowerCAmelCase__       ,		truncation='longest_first'       ,		padding='longest'    )











from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__      :			Optional[Any]           =					logging.get_logger(__name__)

lowercase__      :			Optional[int]           =					{
    "uw-madison/yoso-4096": "https://huggingface.co/uw-madison/yoso-4096/resolve/main/config.json",
    # See all YOSO models at https://huggingface.co/models?filter=yoso
}




class 			a__       (					UpperCamelCase__      ):
     a		:			Dict							=	"""yoso"""







     def __init__(	self  ,		A=50265  ,		A=768  ,		A=12  ,		A=12  ,		A=3072  ,		A="gelu"  ,		A=0.1  ,		A=0.1  ,		A=4096  ,		A=1  ,		A=0.0_2  ,		A=1e-12  ,		A="absolute"  ,		A=True  ,		A=9  ,		A=64  ,		A=None  ,		A=True  ,		A=True  ,		A=1  ,		A=0  ,		A=2  ,		**A  ,		)  ->    Dict:



          '''simple docstring'''



          super().__init__(pad_token_id=A  ,		bos_token_id=A  ,		eos_token_id=A  ,		**A )

          a      =      vocab_size
          a      =      max_position_embeddings
          a      =      hidden_size
          a      =      num_hidden_layers
          a      =      num_attention_heads
          a      =      intermediate_size
          a      =      hidden_act
          a      =      hidden_dropout_prob
          a      =      attention_probs_dropout_prob
          a      =      initializer_range
          a      =      type_vocab_size
          a      =      layer_norm_eps
          a      =      position_embedding_type
          a      =      use_expectation
          a      =      hash_code_len
          a      =      num_hash
          a      =      conv_window
          a      =      use_fast_hash
          a      =      lsh_backward

