


'''simple docstring'''



import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A		=get_tests_dir('fixtures/test_sentencepiece.model')

if is_sentencepiece_available():
		import sentencepiece as sp


__A		=5
__A		=10


@require_sentencepiece
@require_tokenizers
class 						_snake_case   (       a__    ,				unittest.TestCase		):
					lowerCAmelCase :str						=   SpeechaTextTokenizer
					lowerCAmelCase :int						=   False
					lowerCAmelCase :Optional[Any]						=   True




					def 			snake_case__  ( self):
								super().setUp()

								UpperCAmelCase__					:   List[str]       				=       sp.SentencePieceProcessor()
								spm_model.Load(_lowerCamelCase)
								UpperCAmelCase__					:   Optional[int]       				=       ["""<s>""", """<pad>""", """</s>""", """<unk>"""]

								vocab += [spm_model.IdToPiece(id_) for id_ in range(len(_lowerCamelCase))]
								UpperCAmelCase__					:   Optional[int]       				=       dict(zip(_lowerCamelCase							,							range(len(_lowerCamelCase))))

								UpperCAmelCase__					:   int       				=       Path(self.tmpdirname)
								save_json(_lowerCamelCase							,							save_dir / VOCAB_FILES_NAMES["""vocab_file"""])
								if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
											copyfile(_lowerCamelCase							,							save_dir / VOCAB_FILES_NAMES["""spm_file"""])

								UpperCAmelCase__					:   List[Any]       				=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
								tokenizer.save_pretrained(self.tmpdirname)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[Any]       				=       """<pad>"""
								UpperCAmelCase__					:   int       				=       1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase)							,							_lowerCamelCase)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase)							,							_lowerCamelCase)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       list(self.get_tokenizer().get_vocab().keys())

								self.assertEqual(vocab_keys[0]							,							"""<s>""")
								self.assertEqual(vocab_keys[1]							,							"""<pad>""")
								self.assertEqual(vocab_keys[-1]							,							"""j""")
								self.assertEqual(len(_lowerCamelCase)							,							1001)




					def 			snake_case__  ( self):
								self.assertEqual(self.get_tokenizer().vocab_size							,							1001)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

								UpperCAmelCase__					:   List[Any]       				=       tokenizer.tokenize("""This is a test""")
								self.assertListEqual(_lowerCamelCase							,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[289, 50, 14, 174, 386]							,							)

								UpperCAmelCase__					:   Any       				=       tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
								self.assertListEqual(
								    _lowerCamelCase							,							[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]							,							)
								UpperCAmelCase__					:   List[str]       				=       tokenizer.convert_tokens_to_ids(_lowerCamelCase)
								self.assertListEqual(_lowerCamelCase							,							[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8])

								UpperCAmelCase__					:   int       				=       tokenizer.convert_ids_to_tokens(_lowerCamelCase)
								self.assertListEqual(
								    _lowerCamelCase							,							[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]							,							)




					@slow
					def 			snake_case__  ( self):
								# fmt: off
								UpperCAmelCase__					:   Optional[int]       				=       {"""input_ids""": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=_lowerCamelCase							,							model_name="""facebook/s2t-small-mustc-en-de-st"""							,							revision="""a14f04cf0776c02f62a8cb800cf7909e15ea23ad"""							,							)


@require_sentencepiece
class 						_snake_case   (       unittest.TestCase		):
					lowerCAmelCase :Optional[Any]						=   '''valhalla/s2t_mustc_multilinguial_medium'''

					lowerCAmelCase :Tuple						=   '''C\'est trop cool'''
					lowerCAmelCase :Tuple						=   '''Esto es genial'''




					@classmethod
					def 			snake_case__  ( cls):
								UpperCAmelCase__					:   SpeechaTextTokenizer       				=       SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
								return cls




					def 			snake_case__  ( self):
								self.assertEqual(self.tokenizer.lang_code_to_id["""pt"""]							,							4)
								self.assertEqual(self.tokenizer.lang_code_to_id["""ru"""]							,							6)
								self.assertEqual(self.tokenizer.lang_code_to_id["""it"""]							,							9)
								self.assertEqual(self.tokenizer.lang_code_to_id["""de"""]							,							11)




					def 			snake_case__  ( self):
								self.assertEqual(self.tokenizer.vocab_size							,							1_0000)




					def 			snake_case__  ( self):
								self.assertIn(_lowerCamelCase							,							self.tokenizer.all_special_ids)
								UpperCAmelCase__					:   Union[str, Any]       				=       [ES_CODE, 4, 1601, 47, 7647, 2]
								UpperCAmelCase__					:   Any       				=       self.tokenizer.decode(_lowerCamelCase							,							skip_special_tokens=_lowerCamelCase)
								UpperCAmelCase__					:   Any       				=       self.tokenizer.decode(generated_ids[1:]							,							skip_special_tokens=_lowerCamelCase)
								self.assertEqual(_lowerCamelCase							,							_lowerCamelCase)
								self.assertNotIn(self.tokenizer.eos_token							,							_lowerCamelCase)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Union[str, Any]       				=       """fr"""
								UpperCAmelCase__					:   Tuple       				=       self.tokenizer(self.french_text).input_ids
								self.assertEqual(encoded[0]							,							_lowerCamelCase)
								self.assertEqual(encoded[-1]							,							self.tokenizer.eos_token_id)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   List[str]       				=       """fr"""
								self.assertListEqual(self.tokenizer.prefix_tokens							,							[FR_CODE])

								UpperCAmelCase__					:   List[Any]       				=       """es"""
								self.assertListEqual(self.tokenizer.prefix_tokens							,							[ES_CODE])



'''simple docstring'''



import unittest

from .lib import (
    Matrix,
    Vector,
    axpy,
    square_zero_matrix,
    unit_basis_vector,
    zero_vector,
)


class 						_snake_case   (       unittest.TestCase		):




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       Vector([1, 2, 3])
								self.assertEqual(x.component(0)							,							1)
								self.assertEqual(x.component(2)							,							3)
								UpperCAmelCase__					:   List[str]       				=       Vector()




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       Vector([0, 0, 0, 0, 0, 1])
								self.assertEqual(str(_lowerCamelCase)							,							"""(0,0,0,0,0,1)""")




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[int]       				=       Vector([1, 2, 3, 4])
								self.assertEqual(len(_lowerCamelCase)							,							4)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   List[str]       				=       Vector([1, 2])
								UpperCAmelCase__					:   Optional[int]       				=       Vector([1, 2, 3, 4, 5])
								UpperCAmelCase__					:   Optional[Any]       				=       Vector([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
								UpperCAmelCase__					:   Union[str, Any]       				=       Vector([1, -1, 1, -1, 2, -3, 4, -5])
								self.assertAlmostEqual(x.euclidean_length()							,							2.236							,							3)
								self.assertAlmostEqual(y.euclidean_length()							,							7.416							,							3)
								self.assertEqual(z.euclidean_length()							,							0)
								self.assertAlmostEqual(w.euclidean_length()							,							7.616							,							3)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   int       				=       Vector([1, 2, 3])
								UpperCAmelCase__					:   Optional[Any]       				=       Vector([1, 1, 1])
								self.assertEqual((x + y).component(0)							,							2)
								self.assertEqual((x + y).component(1)							,							3)
								self.assertEqual((x + y).component(2)							,							4)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[int]       				=       Vector([1, 2, 3])
								UpperCAmelCase__					:   Dict       				=       Vector([1, 1, 1])
								self.assertEqual((x - y).component(0)							,							0)
								self.assertEqual((x - y).component(1)							,							1)
								self.assertEqual((x - y).component(2)							,							2)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Tuple       				=       Vector([1, 2, 3])
								UpperCAmelCase__					:   Optional[int]       				=       Vector([2, -1, 4])  # for test of dot product
								UpperCAmelCase__					:   Any       				=       Vector([1, -2, -1])
								self.assertEqual(str(x * 3.0)							,							"""(3.0,6.0,9.0)""")
								self.assertEqual((a * b)							,							0)




					def 			snake_case__  ( self):
								self.assertEqual(str(zero_vector(10)).count("""0""")							,							10)




					def 			snake_case__  ( self):
								self.assertEqual(str(unit_basis_vector(3							,							1))							,							"""(0,1,0)""")




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       Vector([1, 2, 3])
								UpperCAmelCase__					:   List[str]       				=       Vector([1, 0, 1])
								self.assertEqual(str(axpy(2							,							_lowerCamelCase							,							_lowerCamelCase))							,							"""(3,4,7)""")




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[int]       				=       Vector([1, 0, 0, 0, 0, 0])
								UpperCAmelCase__					:   Optional[int]       				=       x.copy()
								self.assertEqual(str(_lowerCamelCase)							,							str(_lowerCamelCase))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       Vector([1, 0, 0])
								x.change_component(0							,							0)
								x.change_component(1							,							1)
								self.assertEqual(str(_lowerCamelCase)							,							"""(0,1,0)""")




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								self.assertEqual("""|1,2,3|\n|2,4,5|\n|6,7,8|\n"""							,							str(_lowerCamelCase))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   List[str]       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								UpperCAmelCase__					:   Dict       				=       [[-3, -14, -10], [-5, -10, -5], [-2, -1, 0]]
								for x in range(a.height()):
											for y in range(a.width()):
														self.assertEqual(minors[x][y]							,							a.minor(_lowerCamelCase							,							_lowerCamelCase))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								UpperCAmelCase__					:   Tuple       				=       [[-3, 14, -10], [5, -10, 5], [-2, 1, 0]]
								for x in range(a.height()):
											for y in range(a.width()):
														self.assertEqual(cofactors[x][y]							,							a.cofactor(_lowerCamelCase							,							_lowerCamelCase))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								self.assertEqual(-5							,							a.determinant())




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]							,							3							,							3)
								UpperCAmelCase__					:   List[Any]       				=       Vector([1, 2, 3])
								self.assertEqual("""(14,32,50)"""							,							str(a * x))
								self.assertEqual("""|2,4,6|\n|8,10,12|\n|14,16,18|\n"""							,							str(a * 2))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Union[str, Any]       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								a.change_component(0							,							2							,							5)
								self.assertEqual("""|1,2,5|\n|2,4,5|\n|6,7,8|\n"""							,							str(_lowerCamelCase))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Any       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								self.assertEqual(7							,							a.component(2							,							1)							,							0.01)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								UpperCAmelCase__					:   List[Any]       				=       Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]							,							3							,							3)
								self.assertEqual("""|2,4,10|\n|4,8,10|\n|12,14,18|\n"""							,							str(a + b))




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   List[Any]       				=       Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]							,							3							,							3)
								UpperCAmelCase__					:   Dict       				=       Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]							,							3							,							3)
								self.assertEqual("""|0,0,-4|\n|0,0,0|\n|0,0,-2|\n"""							,							str(a - b))




					def 			snake_case__  ( self):
								self.assertEqual(
								    """|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n"""							,							str(square_zero_matrix(5))							,							)


if __name__ == "__main__":
		unittest.main()