




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class   __a(     _UpperCamelCase				,						unittest.TestCase   ):






    """simple docstring"""





    lowerCAmelCase  			=      PreTrainedTokenizerFast
    lowerCAmelCase  			=      False
    lowerCAmelCase  			=      True
    lowerCAmelCase  			=      '''tokenizer_file'''





    def 							a__      (      self  ) ->		Optional[int]:
         UpperCAmelCase_			:							Dict   =			False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         UpperCAmelCase_			:							str   =			True

         UpperCAmelCase_			:							List[Any]   =			['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
         UpperCAmelCase_			:							Tuple   =			'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         UpperCAmelCase_			:							Any   =			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         UpperCAmelCase_			:							List[Any]   =			PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
         tokenizer.save_pretrained(self.tmpdirname  )





    def 							a__      (      self  ) ->		Tuple:
         pass





    def 							a__      (      self  ) ->		Union[str, Any]:
         pass





    def 							a__      (      self  ) ->		Dict:
         pass





    def 							a__      (      self  ) ->		Tuple:
         pass





    def 							a__      (      self  ) ->		List[Any]:
         UpperCAmelCase_			:							str   =			self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   try:
                        UpperCAmelCase_			:							Optional[int]   =			tempfile.mkdtemp()
                        UpperCAmelCase_			:							List[str]   =			self.rust_tokenizer_class.from_pretrained(a__   ,**a__  )

                        tokenizer.save_pretrained(self.tmpdirname  )
                        super().test_training_new_tokenizer()
                   finally:
                        # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                        # is restored
                        shutil.rmtree(self.tmpdirname  )
                        UpperCAmelCase_			:							str   =			tmpdirname_orig





    def 							a__      (      self  ) ->		Dict:
         UpperCAmelCase_			:							Dict   =			self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   try:
                        UpperCAmelCase_			:							Optional[int]   =			tempfile.mkdtemp()
                        UpperCAmelCase_			:							List[str]   =			self.rust_tokenizer_class.from_pretrained(a__   ,**a__  )

                        tokenizer.save_pretrained(self.tmpdirname  )
                        super().test_training_new_tokenizer_with_special_tokens_change()
                   finally:
                        # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                        # is restored
                        shutil.rmtree(self.tmpdirname  )
                        UpperCAmelCase_			:							Dict   =			tmpdirname_orig





    def 							a__      (      self  ) ->		Optional[Any]:
         UpperCAmelCase_			:							List[Any]   =			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

         UpperCAmelCase_			:							int   =			('''a''' for _ in range(1_000  ))
         UpperCAmelCase_			:							Any   =			tokenizer.train_new_from_iterator(text_iterator=a__   ,length=1_000   ,vocab_size=50  )

         UpperCAmelCase_			:							Union[str, Any]   =			new_tokenizer.encode('''a🤗'''  )
         self.assertEqual(a__   ,[64, 172, 253, 97, 245]  )


@require_tokenizers
class   __a(     unittest.TestCase   ):






    """simple docstring"""





    def 							a__      (      self  ) ->		Optional[Any]:
         UpperCAmelCase_			:							Dict   =			AutoTokenizer.from_pretrained('''bert-base-cased'''  )
         UpperCAmelCase_			:							Dict   =			json.loads(tokenizer._tokenizer.to_str()  )
         UpperCAmelCase_			:							Tuple   =			len(a__  )

         with tempfile.TemporaryDirectory() as tmp_dir:
              # Hack to save this in the tokenizer_config.json
              UpperCAmelCase_			:							int   =			['''tokenizer.4.0.0.json''']
              tokenizer.save_pretrained(a__  )
              json.dump(a__   ,open(os.path.join(a__   ,'''tokenizer.4.0.0.json'''  )   ,'''w'''  )  )

              # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
              UpperCAmelCase_			:							int   =			AutoTokenizer.from_pretrained(a__  )
              self.assertEqual(len(a__  )   ,len(a__  ) + 1  )
              UpperCAmelCase_			:							List[str]   =			json.loads(new_tokenizer._tokenizer.to_str()  )
              self.assertIn('''huggingface'''   ,json_tokenizer['''model''']['''vocab''']  )

              # Will need to be adjusted if we reach v42 and this test is still here.
              # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
              shutil.move(os.path.join(a__   ,'''tokenizer.4.0.0.json'''  )   ,os.path.join(a__   ,'''tokenizer.42.0.0.json'''  )  )
              UpperCAmelCase_			:							Optional[Any]   =			['''tokenizer.42.0.0.json''']
              tokenizer.save_pretrained(a__  )
              UpperCAmelCase_			:							Any   =			AutoTokenizer.from_pretrained(a__  )
              self.assertEqual(len(a__  )   ,len(a__  )  )
              UpperCAmelCase_			:							Any   =			json.loads(new_tokenizer._tokenizer.to_str()  )
              self.assertNotIn('''huggingface'''   ,json_tokenizer['''model''']['''vocab''']  )





    def 							a__      (      self  ) ->		str:
         UpperCAmelCase_			:							Tuple   =			'''hf-internal-testing/test-two-tokenizers'''

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         UpperCAmelCase_			:							Optional[int]   =			AutoTokenizer.from_pretrained(a__  )
         self.assertEqual(len(a__  )   ,28_997  )
         UpperCAmelCase_			:							str   =			json.loads(tokenizer._tokenizer.to_str()  )
         self.assertIn('''huggingface'''   ,json_tokenizer['''model''']['''vocab''']  )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         UpperCAmelCase_			:							List[str]   =			'''3.0.0'''
         UpperCAmelCase_			:							List[Any]   =			old_transformers.models.auto.AutoTokenizer.from_pretrained(a__  )
         self.assertEqual(len(a__  )   ,28_996  )
         UpperCAmelCase_			:							Dict   =			json.loads(old_tokenizer._tokenizer.to_str()  )
         self.assertNotIn('''huggingface'''   ,json_tokenizer['''model''']['''vocab''']  )


@require_tokenizers
class   __a(     unittest.TestCase   ):






    """simple docstring"""





    def 							a__      (      self  ) ->		int:
         UpperCAmelCase_			:							int   =			PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''  )
         UpperCAmelCase_			:							int   =			'''The Matrix is a 1999 science fiction action film.'''

         with concurrent.futures.ThreadPoolExecutor() as executor:
              UpperCAmelCase_			:							Optional[Any]   =			[executor.submit(self.fetch   ,a__   ,a__  ) for i in range(10  )]
              UpperCAmelCase_			:							List[str]   =			[future.result() for future in futures]
              self.assertEqual(a__   ,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )





    def 							a__      (      self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE  ) ->		Dict:
         return tokenizer.encode(a__   ,truncation='''longest_first'''   ,padding='''longest'''  )



"""simple docstring"""


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
		import torch

		from transformers import CamembertModel
@require_torch
@require_sentencepiece
@require_tokenizers
class snake_case					(     unittest.TestCase):







		@slow
		def        a_       (    self					:				int						)				->  Tuple:





						'''simple docstring'''







						_A         =	CamembertModel.from_pretrained("camembert-base"						)
						model.to(a__						)

						_A         =	torch.tensor(
						    [[5, 1_21, 11, 6_60, 16, 7_30, 2_55_43, 1_10, 83, 6]]			,      device=a__			,      dtype=torch.long			,      )  # J'aime le camembert !
						with torch.no_grad():
										_A         =	model(a__						)["last_hidden_state"]
						_A         =	torch.Size((1, 10, 7_68)						)
						self.assertEqual(output.shape			,      a__						)
						# compare the actual values for a slice.
						_A         =	torch.tensor(
						    [[[-0.0_2_5_4, 0.0_2_3_5, 0.1_0_2_7], [0.0_6_0_6, -0.1_8_1_1, -0.0_4_1_8], [-0.1_5_6_1, -0.1_1_2_7, 0.2_6_8_7]]]			,      device=a__			,      dtype=torch.float			,      )
						# camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
						# camembert.eval()
						# expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()

						self.assertTrue(torch.allclose(output[:, :3, :3]			,      a__			,      atol=1E-4						)						)