





'''simple docstring'''




from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFRoFormerForCausalLM,
      TFRoFormerForMaskedLM,
      TFRoFormerForMultipleChoice,
      TFRoFormerForQuestionAnswering,
      TFRoFormerForSequenceClassification,
      TFRoFormerForTokenClassification,
      TFRoFormerModel,
  )
  from transformers.models.roformer.modeling_tf_roformer import (
      TFRoFormerSelfAttention,
      TFRoFormerSinusoidalPositionalEmbedding,
  )


class 					SCREAMING_SNAKE_CASE       :


     def __init__(		self		,      _UpperCAmelCase		,      _UpperCAmelCase=13		,      _UpperCAmelCase=7		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=99		,      _UpperCAmelCase=32		,      _UpperCAmelCase=2		,      _UpperCAmelCase=4		,      _UpperCAmelCase=37		,      _UpperCAmelCase="gelu"		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=512		,      _UpperCAmelCase=16		,      _UpperCAmelCase=2		,      _UpperCAmelCase=0.02		,      _UpperCAmelCase=3		,      _UpperCAmelCase=4		,      _UpperCAmelCase=None		,      ):

          '''simple docstring'''






          __A		:					Optional[Any]						=			parent
          __A		:					List[Any]						=			13
          __A		:					str						=			7
          __A		:					Optional[Any]						=			True
          __A		:					List[str]						=			True
          __A		:					Union[str, Any]						=			True
          __A		:					Dict						=			True
          __A		:					int						=			99
          __A		:					Optional[Any]						=			32
          __A		:					Tuple						=			2
          __A		:					str						=			4
          __A		:					str						=			37
          __A		:					List[str]						=			'gelu'
          __A		:					str						=			0.1
          __A		:					List[Any]						=			0.1
          __A		:					Optional[Any]						=			512
          __A		:					Any						=			16
          __A		:					Optional[int]						=			2
          __A		:					Dict						=			0.02
          __A		:					Union[str, Any]						=			3
          __A		:					List[Any]						=			4
          __A		:					Dict						=			None


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size)

          __A		:					str						=			None
          if self.use_input_mask:
               __A		:					Optional[Any]						=			random_attention_mask([self.batch_size, self.seq_length])

          __A		:					int						=			None
          if self.use_token_type_ids:
               __A		:					Optional[int]						=			ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size)

          __A		:					str						=			None
          __A		:					str						=			None
          __A		:					Any						=			None
          if self.use_labels:
               __A		:					List[str]						=			ids_tensor([self.batch_size]		,      self.type_sequence_label_size)
               __A		:					Optional[Any]						=			ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels)
               __A		:					int						=			ids_tensor([self.batch_size]		,      self.num_choices)

          __A		:					Optional[int]						=			RoFormerConfig(
              vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      return_dict=_UpperCAmelCase		,      )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Any						=			TFRoFormerModel(config=_UpperCAmelCase)
          __A		:					Optional[int]						=			{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

          __A		:					List[Any]						=			[input_ids, input_mask]
          __A		:					str						=			model(_UpperCAmelCase)

          __A		:					Tuple						=			model(_UpperCAmelCase)

          self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Tuple						=			True
          __A		:					List[Any]						=			TFRoFormerForCausalLM(config=_UpperCAmelCase)
          __A		:					str						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          __A		:					Tuple						=			model(_UpperCAmelCase)['logits']
          self.parent.assertListEqual(
              list(prediction_scores.numpy().shape)		,      [self.batch_size, self.seq_length, self.vocab_size])


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					List[str]						=			TFRoFormerForMaskedLM(config=_UpperCAmelCase)
          __A		:					str						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          __A		:					Union[str, Any]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					List[str]						=			self.num_labels
          __A		:					Tuple						=			TFRoFormerForSequenceClassification(config=_UpperCAmelCase)
          __A		:					Dict						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          __A		:					int						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Optional[Any]						=			self.num_choices
          __A		:					List[str]						=			TFRoFormerForMultipleChoice(config=_UpperCAmelCase)
          __A		:					Optional[Any]						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					Dict						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					Any						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					str						=			{
              'input_ids': multiple_choice_inputs_ids,
              'attention_mask': multiple_choice_input_mask,
              'token_type_ids': multiple_choice_token_type_ids,
          }
          __A		:					List[Any]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					List[str]						=			self.num_labels
          __A		:					List[Any]						=			TFRoFormerForTokenClassification(config=_UpperCAmelCase)
          __A		:					Dict						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          __A		:					List[str]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					str						=			TFRoFormerForQuestionAnswering(config=_UpperCAmelCase)
          __A		:					Union[str, Any]						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          __A		:					List[Any]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length))


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			self.prepare_config_and_inputs()
          (
              (
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,
          )		:					Optional[Any]						=			config_and_inputs
          __A		:					List[Any]						=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
          return config, inputs_dict




@require_tf
class 					SCREAMING_SNAKE_CASE       (a__   ,						a__   ,						unittest.TestCase				):
     lowerCAmelCase						     =			(
         (
             TFRoFormerModel,
             TFRoFormerForCausalLM,
             TFRoFormerForMaskedLM,
             TFRoFormerForQuestionAnswering,
             TFRoFormerForSequenceClassification,
             TFRoFormerForTokenClassification,
             TFRoFormerForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     lowerCAmelCase						     =			(
         {
             '''feature-extraction''': TFRoFormerModel,
             '''fill-mask''': TFRoFormerForMaskedLM,
             '''question-answering''': TFRoFormerForQuestionAnswering,
             '''text-classification''': TFRoFormerForSequenceClassification,
             '''text-generation''': TFRoFormerForCausalLM,
             '''token-classification''': TFRoFormerForTokenClassification,
             '''zero-shot''': TFRoFormerForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )

     lowerCAmelCase						     =			False
     lowerCAmelCase						     =			False


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          if pipeline_test_casse_name == "TextGenerationPipelineTests":
               return True

          return False


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			TFRoFormerModelTester(self)
          __A		:					Tuple						=			ConfigTester(self		,      config_class=_UpperCAmelCase		,      hidden_size=37)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          self.config_tester.run_common_tests()


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[Any]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[Any]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase)


     @slow
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			TFRoFormerModel.from_pretrained('junnyu/roformer_chinese_base')
          self.assertIsNotNone(_UpperCAmelCase)




@require_tf
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     @slow
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			TFRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base')
          __A		:					str						=			tf.constant([[0, 1, 2, 3, 4, 5]])
          __A		:					Tuple						=			model(_UpperCAmelCase)[0]

          # TODO Replace vocab size
          __A		:					Tuple						=			5_0000

          __A		:					Any						=			[1, 6, vocab_size]
          self.assertEqual(output.shape		,      _UpperCAmelCase)

          print(output[:, :3, :3])

          # TODO Replace values below with what was printed above.
          __A		:					int						=			tf.constant(
              [
                  [
                      [-0.12053341, -1.0264901, 0.29221946],
                      [-1.5133783, 0.197433, 0.15190607],
                      [-5.0135403, -3.900256, -0.84038764],
                  ]
              ])
          tf.debugging.assert_near(output[:, :3, :3]		,      _UpperCAmelCase		,      atol=1e-4)




@require_tf
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):
     lowerCAmelCase						     =			1E-4


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			tf.constant([[4, 10]])
          __A		:					Optional[Any]						=			TFRoFormerSinusoidalPositionalEmbedding(num_positions=6		,      embedding_dim=6)

          __A		:					List[str]						=			emba(input_ids.shape)
          __A		:					int						=			tf.constant(
              [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]])

          tf.debugging.assert_near(_UpperCAmelCase		,      _UpperCAmelCase		,      atol=self.tolerance)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[Any]						=			tf.constant(
              [
                  [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                  [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                  [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
              ])
          __A		:					Any						=			TFRoFormerSinusoidalPositionalEmbedding(num_positions=512		,      embedding_dim=512)
          emba([2, 16, 512])
          __A		:					Tuple						=			emba.weight[:3, :5]

          tf.debugging.assert_near(_UpperCAmelCase		,      _UpperCAmelCase		,      atol=self.tolerance)




@require_tf
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):
     lowerCAmelCase						     =			1E-4


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Any						=			tf.reshape(tf.range(2 * 12 * 16 * 64		,      dtype=tf.floataa)		,      shape=(2, 12, 16, 64)) / 100

          __A		:					Dict						=			-tf.reshape(tf.range(2 * 12 * 16 * 64		,      dtype=tf.floataa)		,      shape=(2, 12, 16, 64)) / 100

          __A		:					List[Any]						=			TFRoFormerSinusoidalPositionalEmbedding(num_positions=32		,      embedding_dim=64)
          __A		:					str						=			embed_positions([2, 16, 768])[None, None, :, :]

          __A					,__A		:					Tuple						=			TFRoFormerSelfAttention.apply_rotary_position_embeddings(
              _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase)

          __A		:					Dict						=			tf.constant(
              [
                  [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                  [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                  [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                  [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                  [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                  [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
              ])
          __A		:					Optional[int]						=			tf.constant(
              [
                  [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                  [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                  [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                  [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                  [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                  [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
              ])

          tf.debugging.assert_near(query_layer[0, 0, :6, :8]		,      _UpperCAmelCase		,      atol=self.tolerance)
          tf.debugging.assert_near(key_layer[0, 0, :6, :8]		,      _UpperCAmelCase		,      atol=self.tolerance)





import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase      :						Dict							=  get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
  import sentencepiece as sp


_lowerCamelCase      :						str							=  5
_lowerCamelCase      :						int							=  10


@require_sentencepiece
@require_tokenizers
class 							UpperCamelCase_       (					UpperCAmelCase__						,   unittest.TestCase	):

    '''simple docstring'''






    UpperCAmelCase__				    = SpeechaTextTokenizer
    UpperCAmelCase__				    = False
    UpperCAmelCase__				    = True


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->List[str]:





           '''simple docstring'''

           super().setUp()

           A__ 					=       sp.SentencePieceProcessor()
           spm_model.Load(UpperCAmelCase__)
           A__ 					=       ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

           vocab += [spm_model.IdToPiece(id_) for id_ in range(len(UpperCAmelCase__))]
           A__ 					=       dict(zip(UpperCAmelCase__   ,					range(len(UpperCAmelCase__))))

           A__ 					=       Path(self.tmpdirname)
           save_json(UpperCAmelCase__   ,					save_dir / VOCAB_FILES_NAMES['''vocab_file'''])
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                  copyfile(UpperCAmelCase__   ,					save_dir / VOCAB_FILES_NAMES['''spm_file'''])

           A__ 					=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
           tokenizer.save_pretrained(self.tmpdirname)


    def 			SCREAMING_SNAKE_CASE				(     self :      Any)		->Optional[int]:





           '''simple docstring'''

           A__ 					=       '''<pad>'''
           A__ 					=       1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__)   ,					UpperCAmelCase__)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__)   ,					UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Any:





           '''simple docstring'''

           A__ 					=       list(self.get_tokenizer().get_vocab().keys())

           self.assertEqual(vocab_keys[0]   ,					'''<s>''')
           self.assertEqual(vocab_keys[1]   ,					'''<pad>''')
           self.assertEqual(vocab_keys[-1]   ,					'''j''')
           self.assertEqual(len(UpperCAmelCase__)   ,					1_001)


    def 			SCREAMING_SNAKE_CASE				(     self :      List[Any])		->List[str]:





           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size   ,					1_001)


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->List[str]:





           '''simple docstring'''

           A__ 					=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

           A__ 					=       tokenizer.tokenize('''This is a test''')
           self.assertListEqual(UpperCAmelCase__   ,					['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCAmelCase__)   ,					[289, 50, 14, 174, 386]   ,					)

           A__ 					=       tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
           self.assertListEqual(
               UpperCAmelCase__   ,					[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']   ,					)
           A__ 					=       tokenizer.convert_tokens_to_ids(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__   ,					[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8])

           A__ 					=       tokenizer.convert_ids_to_tokens(UpperCAmelCase__)
           self.assertListEqual(
               UpperCAmelCase__   ,					[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']   ,					)





    @slow
    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->List[Any]:





           '''simple docstring'''

           A__ 					=       {'''input_ids''': [[3_791, 797, 31, 11, 64, 797, 31, 2_429, 433, 12, 1_176, 12, 20, 786, 915, 142, 2_413, 240, 37, 3_238, 797, 31, 11, 35, 93, 915, 142, 2_413, 240, 37, 5_540, 567, 1_276, 93, 37, 610, 40, 62, 455, 657, 1_042, 123, 780, 177, 37, 309, 241, 1_298, 514, 20, 292, 2_737, 114, 2_469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3_388, 511, 459, 4, 3_555, 40, 321, 302, 705, 4, 3_388, 511, 583, 326, 5, 5, 5, 62, 3_310, 560, 177, 2_680, 217, 1_508, 32, 31, 853, 418, 64, 583, 511, 1_605, 62, 35, 93, 560, 177, 2_680, 217, 1_508, 1_521, 64, 583, 511, 519, 62, 20, 1_515, 764, 20, 149, 261, 5_625, 7_972, 20, 5_540, 567, 1_276, 93, 3_925, 1_675, 11, 15, 802, 7_972, 576, 217, 1_508, 11, 35, 93, 1_253, 2_441, 15, 289, 652, 31, 416, 321, 3_842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2_681, 1_153, 3_434, 20, 5_540, 37, 567, 126, 1_253, 2_441, 3_376, 449, 210, 431, 1_563, 177, 767, 5_540, 11, 1_203, 472, 11, 2_953, 685, 285, 364, 706, 1_153, 20, 6_799, 20, 2_869, 20, 4_464, 126, 40, 2_429, 20, 1_040, 866, 2_664, 418, 20, 318, 20, 1_726, 186, 20, 265, 522, 35, 93, 2_191, 4_634, 20, 1_040, 12, 6_799, 15, 228, 2_356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_575, 2_666, 684, 1_582, 1_176, 12, 627, 149, 619, 20, 4_902, 563, 11, 20, 149, 261, 3_420, 2_356, 174, 142, 4_714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCAmelCase__   ,					model_name='''facebook/s2t-small-mustc-en-de-st'''   ,					revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''   ,					)


@require_sentencepiece
class 							UpperCamelCase_       (					unittest.TestCase	):

    '''simple docstring'''






    UpperCAmelCase__				    = '''valhalla/s2t_mustc_multilinguial_medium'''

    UpperCAmelCase__				    = '''C\'est trop cool'''
    UpperCAmelCase__				    = '''Esto es genial'''


    @classmethod
    def 			SCREAMING_SNAKE_CASE				(     cls :      Dict)		->Dict:





           '''simple docstring'''

           A__ 					=       SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
           return cls


    def 			SCREAMING_SNAKE_CASE				(     self :      str)		->Dict:





           '''simple docstring'''

           self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']   ,					4)
           self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']   ,					6)
           self.assertEqual(self.tokenizer.lang_code_to_id['''it''']   ,					9)
           self.assertEqual(self.tokenizer.lang_code_to_id['''de''']   ,					11)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Dict:





           '''simple docstring'''

           self.assertEqual(self.tokenizer.vocab_size   ,					10_000)


    def 			SCREAMING_SNAKE_CASE				(     self :      Dict)		->Union[str, Any]:





           '''simple docstring'''

           self.assertIn(UpperCAmelCase__   ,					self.tokenizer.all_special_ids)
           A__ 					=       [ES_CODE, 4, 1_601, 47, 7_647, 2]
           A__ 					=       self.tokenizer.decode(UpperCAmelCase__   ,					skip_special_tokens=UpperCAmelCase__)
           A__ 					=       self.tokenizer.decode(generated_ids[1:]   ,					skip_special_tokens=UpperCAmelCase__)
           self.assertEqual(UpperCAmelCase__   ,					UpperCAmelCase__)
           self.assertNotIn(self.tokenizer.eos_token   ,					UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->str:





           '''simple docstring'''

           A__ 					=       '''fr'''
           A__ 					=       self.tokenizer(self.french_text).input_ids
           self.assertEqual(encoded[0]   ,					UpperCAmelCase__)
           self.assertEqual(encoded[-1]   ,					self.tokenizer.eos_token_id)





    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->int:





           '''simple docstring'''

           A__ 					=       '''fr'''
           self.assertListEqual(self.tokenizer.prefix_tokens   ,					[FR_CODE])

           A__ 					=       '''es'''
           self.assertListEqual(self.tokenizer.prefix_tokens   ,					[ES_CODE])


