

'''simple docstring'''





import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


A_	= get_tests_dir("fixtures/test_sentencepiece.model")
A_	= get_tests_dir("fixtures/test_sentencepiece_bpe.model")

A_	= "pt" if is_torch_available() else "tf"


@require_sentencepiece
@require_tokenizers
class  UpperCAmelCase (							UpperCAmelCase__  , unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					CamembertTokenizer
		SCREAMING_SNAKE_CASE_           =					CamembertTokenizerFast
		SCREAMING_SNAKE_CASE_           =					True
		SCREAMING_SNAKE_CASE_           =					True
		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCamelCase_  				=      CamembertTokenizer(SCREAMING_SNAKE_CASE_		)
							tokenizer.save_pretrained(self.tmpdirname		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      '<pad>'
							lowerCamelCase_  				=      1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_		)   ,			SCREAMING_SNAKE_CASE_		)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_		)   ,			SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      list(self.get_tokenizer().get_vocab().keys()		)

							self.assertEqual(vocab_keys[0]   ,			'<s>NOTUSED'		)
							self.assertEqual(vocab_keys[1]   ,			'<pad>'		)
							self.assertEqual(vocab_keys[-1]   ,			'<mask>'		)
							self.assertEqual(len(SCREAMING_SNAKE_CASE_		)   ,			1004		)
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							self.assertEqual(self.get_tokenizer().vocab_size   ,			1005		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      CamembertTokenizer(SCREAMING_SNAKE_CASE_		)
							tokenizer.save_pretrained(self.tmpdirname		)
							lowerCamelCase_  				=      CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

							lowerCamelCase_  				=      'I was born in 92000, and this is falsé.'

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							# <unk> tokens are not the same for `rust` than for `slow`.
							# Because spm gives back raw token instead of `unk` in EncodeAsPieces
							# tokens = tokenizer.tokenize(sequence)
							lowerCamelCase_  				=      tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							if not self.test_rust_tokenizer:
												return

							lowerCamelCase_  				=      self.get_tokenizer()
							lowerCamelCase_  				=      self.get_rust_tokenizer()

							lowerCamelCase_  				=      'I was born in 92000, and this is falsé.'

							lowerCamelCase_  				=      tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      self.get_rust_tokenizer()
							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)






		@slow
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      {'input_ids': [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 27575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 22804, 18818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 10326, 24, 2267, 20, 416, 5072, 15612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							# camembert is a french model. So we also use french texts.
							lowerCamelCase_  				=      [
							    'Le transformeur est un modèle d\'apprentissage profond introduit en 2017, '
							    'utilisé principalement dans le domaine du traitement automatique des langues (TAL).',
							    'À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus '
							    'pour gérer des données séquentielles, telles que le langage naturel, pour des tâches '
							    'telles que la traduction et la synthèse de texte.',
							]

							self.tokenizer_integration_test_util(
							    expected_encoding=SCREAMING_SNAKE_CASE_   ,			model_name='camembert-base'   ,			revision='3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'   ,			sequences=SCREAMING_SNAKE_CASE_   ,			)








"""simple docstring"""




import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_torch_available
from transformers.testing_utils import require_torch, torch_device


if is_torch_available():
 from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments






@require_torch
class       UpperCAmelCase_    (							unittest.TestCase		):



      def 		snake_case_   (	self	:      Union[str, Any]	,      A	:      Optional[Any]		):
           for model_result in results.values():
                for batch_size, sequence_length in zip(model_result["bs"]	,      model_result["ss"]		):
                     _UpperCAmelCase     :      List[str]														=       model_result["result"][batch_size][sequence_length]
                     self.assertIsNotNone(A		)



      def 		snake_case_   (	self	:      Optional[Any]		):
           _UpperCAmelCase     :      Any														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      int														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      List[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      Dict		):
           _UpperCAmelCase     :      List[str]														=       "sgugger/tiny-distilbert-classification"
           _UpperCAmelCase     :      Optional[int]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      only_pretrain_model=A	,      )
           _UpperCAmelCase     :      Dict														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      Optional[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      Any														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      torchscript=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      Dict														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      Optional[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      @unittest.skipIf(torch_device == "cpu"	,      "Cant do half precision"		)
      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      Union[str, Any]														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      Optional[Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      fpaa=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      List[str]														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      List[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Optional[Any]														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      int														=       AutoConfig.from_pretrained(A		)
           # set architectures equal to `None`
           _UpperCAmelCase     :      Optional[Any]														=       None
           _UpperCAmelCase     :      Optional[Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmark(A	,      configs=[config]		)
           _UpperCAmelCase     :      Optional[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      Dict														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      Union[str, Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      Tuple														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      Union[str, Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_train_result		)
           self.check_results_dict_not_empty(results.memory_train_result		)



      @unittest.skipIf(torch_device == "cpu"	,      "Can't do half precision"		)
      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Optional[int]														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      str														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      fpaa=A	,      multi_process=A	,      )
           _UpperCAmelCase     :      Tuple														=       PyTorchBenchmark(A		)
           _UpperCAmelCase     :      List[Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_train_result		)
           self.check_results_dict_not_empty(results.memory_train_result		)



      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Optional[Any]														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      Union[str, Any]														=       AutoConfig.from_pretrained(A		)
           _UpperCAmelCase     :      Dict														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmark(A	,      configs=[config]		)
           _UpperCAmelCase     :      List[str]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      str		):
           _UpperCAmelCase     :      List[str]														=       "sshleifer/tinier_bart"
           _UpperCAmelCase     :      Any														=       AutoConfig.from_pretrained(A		)
           _UpperCAmelCase     :      Optional[int]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      int														=       PyTorchBenchmark(A	,      configs=[config]		)
           _UpperCAmelCase     :      Union[str, Any]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_inference_result		)
           self.check_results_dict_not_empty(results.memory_inference_result		)



      def 		snake_case_   (	self	:      Any		):
           _UpperCAmelCase     :      Tuple														=       "sshleifer/tiny-gpt2"
           _UpperCAmelCase     :      Tuple														=       AutoConfig.from_pretrained(A		)
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      int														=       PyTorchBenchmark(A	,      configs=[config]		)
           _UpperCAmelCase     :      Optional[int]														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_train_result		)
           self.check_results_dict_not_empty(results.memory_train_result		)



      def 		snake_case_   (	self	:      str		):
           _UpperCAmelCase     :      Optional[int]														=       "sshleifer/tinier_bart"
           _UpperCAmelCase     :      int														=       AutoConfig.from_pretrained(A		)
           _UpperCAmelCase     :      List[Any]														=       PyTorchBenchmarkArguments(
               models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      multi_process=A	,      )
           _UpperCAmelCase     :      str														=       PyTorchBenchmark(A	,      configs=[config]		)
           _UpperCAmelCase     :      int														=       benchmark.run()
           self.check_results_dict_not_empty(results.time_train_result		)
           self.check_results_dict_not_empty(results.memory_train_result		)



      def 		snake_case_   (	self	:      Dict		):
           _UpperCAmelCase     :      int														=       "sshleifer/tiny-gpt2"
           with tempfile.TemporaryDirectory() as tmp_dir:
                _UpperCAmelCase     :      Union[str, Any]														=       PyTorchBenchmarkArguments(
                    models=[MODEL_ID]	,      training=A	,      inference=A	,      save_to_csv=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      inference_time_csv_file=os.path.join(A	,      "inf_time.csv"		)	,      train_memory_csv_file=os.path.join(A	,      "train_mem.csv"		)	,      inference_memory_csv_file=os.path.join(A	,      "inf_mem.csv"		)	,      train_time_csv_file=os.path.join(A	,      "train_time.csv"		)	,      env_info_csv_file=os.path.join(A	,      "env.csv"		)	,      multi_process=A	,      )
                _UpperCAmelCase     :      int														=       PyTorchBenchmark(A		)
                benchmark.run()
                self.assertTrue(Path(os.path.join(A	,      "inf_time.csv"		)		).exists()		)
                self.assertTrue(Path(os.path.join(A	,      "train_time.csv"		)		).exists()		)
                self.assertTrue(Path(os.path.join(A	,      "inf_mem.csv"		)		).exists()		)
                self.assertTrue(Path(os.path.join(A	,      "train_mem.csv"		)		).exists()		)
                self.assertTrue(Path(os.path.join(A	,      "env.csv"		)		).exists()		)



      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      List[Any]														=       "sshleifer/tiny-gpt2"

           def _check_summary_is_not_empty(A	:      int		):
                self.assertTrue(hasattr(A	,      "sequential"		)		)
                self.assertTrue(hasattr(A	,      "cumulative"		)		)
                self.assertTrue(hasattr(A	,      "current"		)		)
                self.assertTrue(hasattr(A	,      "total"		)		)

           with tempfile.TemporaryDirectory() as tmp_dir:
                _UpperCAmelCase     :      int														=       PyTorchBenchmarkArguments(
                    models=[MODEL_ID]	,      training=A	,      inference=A	,      sequence_lengths=[8]	,      batch_sizes=[1]	,      log_filename=os.path.join(A	,      "log.txt"		)	,      log_print=A	,      trace_memory_line_by_line=A	,      multi_process=A	,      )
                _UpperCAmelCase     :      int														=       PyTorchBenchmark(A		)
                _UpperCAmelCase     :      Any														=       benchmark.run()
                _check_summary_is_not_empty(result.inference_summary		)
                _check_summary_is_not_empty(result.train_summary		)
                self.assertTrue(Path(os.path.join(A	,      "log.txt"		)		).exists()		)
