
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowercase   =	get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')





@require_sentencepiece
@require_tokenizers
class    __A(					UpperCAmelCase  ,		unittest.TestCase							):
					SCREAMING_SNAKE_CASE           =     SpeechTaTokenizer
					SCREAMING_SNAKE_CASE           =     False
					SCREAMING_SNAKE_CASE           =     True

					def  lowercase__   (						self       :  Any ):
												super().setUp()

												# We have a SentencePiece fixture for testing
												lowerCamelCase_					=       SpeechTaTokenizer(__UpperCamelCase )

												lowerCamelCase_					=       AddedToken("""<mask>""" ,       lstrip=__UpperCamelCase ,       rstrip=__UpperCamelCase )
												lowerCamelCase_					=       mask_token
												tokenizer.add_special_tokens({"""mask_token""": mask_token} )
												tokenizer.add_tokens(["""<ctc_blank>"""] )

												tokenizer.save_pretrained(self.tmpdirname )

					def  lowercase__   (						self       :  Optional[Any] ,       __UpperCamelCase       :  Any ):
												lowerCamelCase_					=       """this is a test"""
												lowerCamelCase_					=       """this is a test"""
												return input_text, output_text

					def  lowercase__   (						self       :  str ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  Optional[int]=False ,       __UpperCamelCase       :  Optional[int]=2_0 ,       __UpperCamelCase       :  str=5 ):
												lowerCamelCase_	, lowerCamelCase_					=       self.get_input_output_texts(__UpperCamelCase )
												lowerCamelCase_					=       tokenizer.encode(__UpperCamelCase ,       add_special_tokens=__UpperCamelCase )
												lowerCamelCase_					=       tokenizer.decode(__UpperCamelCase ,       clean_up_tokenization_spaces=__UpperCamelCase )
												return text, ids

					def  lowercase__   (						self       :  Tuple ):
												lowerCamelCase_					=       """<pad>"""
												lowerCamelCase_					=       1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase ) ,       __UpperCamelCase )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase ) ,       __UpperCamelCase )

					def  lowercase__   (						self       :  List[Any] ):
												lowerCamelCase_					=       list(self.get_tokenizer().get_vocab().keys() )

												self.assertEqual(vocab_keys[0] ,       """<s>""" )
												self.assertEqual(vocab_keys[1] ,       """<pad>""" )
												self.assertEqual(vocab_keys[-4] ,       """œ""" )
												self.assertEqual(vocab_keys[-2] ,       """<mask>""" )
												self.assertEqual(vocab_keys[-1] ,       """<ctc_blank>""" )
												self.assertEqual(len(__UpperCamelCase ) ,       8_1 )

					def  lowercase__   (						self       :  Any ):
												self.assertEqual(self.get_tokenizer().vocab_size ,       7_9 )

					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_					=       self.get_tokenizers(do_lower_case=__UpperCamelCase )
												for tokenizer in tokenizers:
																			with self.subTest(F'''{tokenizer.__class__.__name__}''' ):
																										lowerCamelCase_					=       tokenizer.vocab_size
																										lowerCamelCase_					=       len(__UpperCamelCase )

																										self.assertNotEqual(__UpperCamelCase ,       0 )

																										# We usually have added tokens from the start in tests because our vocab fixtures are
																										# smaller than the original vocabs - let's not assert this
																										# self.assertEqual(vocab_size, all_size)

																										lowerCamelCase_					=       ["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
																										lowerCamelCase_					=       tokenizer.add_tokens(__UpperCamelCase )
																										lowerCamelCase_					=       tokenizer.vocab_size
																										lowerCamelCase_					=       len(__UpperCamelCase )

																										self.assertNotEqual(__UpperCamelCase ,       0 )
																										self.assertEqual(__UpperCamelCase ,       __UpperCamelCase )
																										self.assertEqual(__UpperCamelCase ,       len(__UpperCamelCase ) )
																										self.assertEqual(__UpperCamelCase ,       all_size + len(__UpperCamelCase ) )

																										lowerCamelCase_					=       tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l""" ,       add_special_tokens=__UpperCamelCase )

																										self.assertGreaterEqual(len(__UpperCamelCase ) ,       4 )
																										self.assertGreater(tokens[0] ,       tokenizer.vocab_size - 1 )
																										self.assertGreater(tokens[-3] ,       tokenizer.vocab_size - 1 )

																										lowerCamelCase_					=       {"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
																										lowerCamelCase_					=       tokenizer.add_special_tokens(__UpperCamelCase )
																										lowerCamelCase_					=       tokenizer.vocab_size
																										lowerCamelCase_					=       len(__UpperCamelCase )

																										self.assertNotEqual(__UpperCamelCase ,       0 )
																										self.assertEqual(__UpperCamelCase ,       __UpperCamelCase )
																										self.assertEqual(__UpperCamelCase ,       len(__UpperCamelCase ) )
																										self.assertEqual(__UpperCamelCase ,       all_size_a + len(__UpperCamelCase ) )

																										lowerCamelCase_					=       tokenizer.encode(
																										    """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l""" ,       add_special_tokens=__UpperCamelCase )

																										self.assertGreaterEqual(len(__UpperCamelCase ) ,       6 )
																										self.assertGreater(tokens[0] ,       tokenizer.vocab_size - 1 )
																										self.assertGreater(tokens[0] ,       tokens[1] )
																										self.assertGreater(tokens[-3] ,       tokenizer.vocab_size - 1 )
																										self.assertGreater(tokens[-3] ,       tokens[-4] )
																										self.assertEqual(tokens[0] ,       tokenizer.eos_token_id )
																										self.assertEqual(tokens[-3] ,       tokenizer.pad_token_id )

					def  lowercase__   (						self       :  Union[str, Any] ):
												pass

					def  lowercase__   (						self       :  Union[str, Any] ):
												pass

					def  lowercase__   (						self       :  Union[str, Any] ):
												lowerCamelCase_					=       self.get_tokenizer()

												lowerCamelCase_					=       tokenizer.tokenize("""This is a test""" )
												# fmt: off
												self.assertListEqual(__UpperCamelCase ,       [SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""] )
												# fmt: on

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__UpperCamelCase ) ,       [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6] ,       )

												lowerCamelCase_					=       tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
												self.assertListEqual(
												    __UpperCamelCase ,       [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""] )

												lowerCamelCase_					=       tokenizer.convert_tokens_to_ids(__UpperCamelCase )
												# fmt: off
												self.assertListEqual(__UpperCamelCase ,       [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6] )
												# fmt: on

												lowerCamelCase_					=       tokenizer.convert_ids_to_tokens(__UpperCamelCase )
												self.assertListEqual(
												    __UpperCamelCase ,       [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""] )



					@slow
					def  lowercase__   (						self       :  int ):
												# Use custom sequence because this tokenizer does not handle numbers.
												lowerCamelCase_					=       [
												    """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
												    """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
												    """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
												    """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
												    """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
												    """conditioning on both left and right context in all layers.""",
												    """The quick brown fox jumps over the lazy dog.""",
												]

												# fmt: off
												lowerCamelCase_					=       {
												    """input_ids""": [
												        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
												        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
												        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
												    ],
												    """attention_mask""": [
												        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
												        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
												        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
												    ]
												}
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=__UpperCamelCase ,       model_name="""microsoft/speecht5_asr""" ,       revision="""c5ef64c71905caeccde0e4462ef3f9077224c524""" ,       sequences=__UpperCamelCase ,       )







import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotSmallConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
		import os

		# The slow tests are often failing with OOM error on GPU
		# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
		# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
		lowercase   =	'''platform'''

		import jax
		import jax.numpy as jnp

		from transformers.models.blenderbot_small.modeling_flax_blenderbot_small import (
		    FlaxBlenderbotSmallForConditionalGeneration,
		    FlaxBlenderbotSmallModel,
		    shift_tokens_right,
		)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Optional[int]      ,					UpperCAmelCase__					:   Union[str, Any]      ,					UpperCAmelCase__					:   Any=None      ,					UpperCAmelCase__					:   Optional[Any]=None      ,					UpperCAmelCase__					:   List[str]=None      ,					UpperCAmelCase__					:   Optional[int]=None      ,					UpperCAmelCase__					:   List[str]=None      ,					UpperCAmelCase__					:   Dict=None      ,					)	->     List[Any]:
							if attention_mask is None:
														lowerCamelCase_					=       np.where(input_ids != config.pad_token_id      ,					1      ,					0							)
							if decoder_attention_mask is None:
														lowerCamelCase_					=       np.where(decoder_input_ids != config.pad_token_id      ,					1      ,					0							)
							if head_mask is None:
														lowerCamelCase_					=       np.ones((config.encoder_layers, config.encoder_attention_heads)							)
							if decoder_head_mask is None:
														lowerCamelCase_					=       np.ones((config.decoder_layers, config.decoder_attention_heads)							)
							if cross_attn_head_mask is None:
														lowerCamelCase_					=       np.ones((config.decoder_layers, config.decoder_attention_heads)							)
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": attention_mask,
							}





class    __A:

					def __init__(						self       :  str ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  str=1_3 ,       __UpperCamelCase       :  List[str]=7 ,       __UpperCamelCase       :  Tuple=True ,       __UpperCamelCase       :  List[Any]=False ,       __UpperCamelCase       :  Dict=9_9 ,       __UpperCamelCase       :  Tuple=1_6 ,       __UpperCamelCase       :  Any=2 ,       __UpperCamelCase       :  Dict=4 ,       __UpperCamelCase       :  List[str]=4 ,       __UpperCamelCase       :  Optional[Any]="gelu" ,       __UpperCamelCase       :  List[str]=0.1 ,       __UpperCamelCase       :  str=0.1 ,       __UpperCamelCase       :  Union[str, Any]=3_2 ,       __UpperCamelCase       :  str=2 ,       __UpperCamelCase       :  Tuple=1 ,       __UpperCamelCase       :  Optional[Any]=0 ,       __UpperCamelCase       :  str=0.02 ,       ):
												lowerCamelCase_					=       parent
												lowerCamelCase_					=       batch_size
												lowerCamelCase_					=       seq_length
												lowerCamelCase_					=       is_training
												lowerCamelCase_					=       use_labels
												lowerCamelCase_					=       vocab_size
												lowerCamelCase_					=       hidden_size
												lowerCamelCase_					=       num_hidden_layers
												lowerCamelCase_					=       num_attention_heads
												lowerCamelCase_					=       intermediate_size
												lowerCamelCase_					=       hidden_act
												lowerCamelCase_					=       hidden_dropout_prob
												lowerCamelCase_					=       attention_probs_dropout_prob
												lowerCamelCase_					=       max_position_embeddings
												lowerCamelCase_					=       eos_token_id
												lowerCamelCase_					=       pad_token_id
												lowerCamelCase_					=       bos_token_id
												lowerCamelCase_					=       initializer_range

					def  lowercase__   (						self       :  str ):
												lowerCamelCase_					=       np.clip(ids_tensor([self.batch_size, self.seq_length - 1] ,       self.vocab_size ) ,       3 ,       self.vocab_size )
												lowerCamelCase_					=       np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1) ,       dtype=np.intaa )) ,       -1 )

												lowerCamelCase_					=       shift_tokens_right(__UpperCamelCase ,       1 ,       2 )

												lowerCamelCase_					=       BlenderbotSmallConfig(
												    vocab_size=self.vocab_size ,       d_model=self.hidden_size ,       encoder_layers=self.num_hidden_layers ,       decoder_layers=self.num_hidden_layers ,       encoder_attention_heads=self.num_attention_heads ,       decoder_attention_heads=self.num_attention_heads ,       encoder_ffn_dim=self.intermediate_size ,       decoder_ffn_dim=self.intermediate_size ,       dropout=self.hidden_dropout_prob ,       attention_dropout=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       eos_token_id=self.eos_token_id ,       bos_token_id=self.bos_token_id ,       pad_token_id=self.pad_token_id ,       initializer_range=self.initializer_range ,       use_cache=__UpperCamelCase ,       )
												lowerCamelCase_					=       prepare_blenderbot_inputs_dict(__UpperCamelCase ,       __UpperCamelCase ,       __UpperCamelCase )
												return config, inputs_dict

					def  lowercase__   (						self       :  List[str] ):
												lowerCamelCase_	, lowerCamelCase_					=       self.prepare_config_and_inputs()
												return config, inputs_dict

					def  lowercase__   (						self       :  Tuple ,       __UpperCamelCase       :  int ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  str ):
												lowerCamelCase_					=       2_0
												lowerCamelCase_					=       model_class_name(__UpperCamelCase )

												lowerCamelCase_					=       model.encode(inputs_dict["""input_ids"""] )

												lowerCamelCase_	, lowerCamelCase_					=       (
												    inputs_dict["""decoder_input_ids"""],
												    inputs_dict["""decoder_attention_mask"""],
												)

												lowerCamelCase_					=       model.init_cache(decoder_input_ids.shape[0] ,       __UpperCamelCase ,       __UpperCamelCase )
												lowerCamelCase_					=       jnp.ones((decoder_input_ids.shape[0], max_decoder_length) ,       dtype="""i4""" )

												lowerCamelCase_					=       jnp.broadcast_to(
												    jnp.arange(decoder_input_ids.shape[-1] - 1 )[None, :] ,       (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1) ,       )
												lowerCamelCase_					=       model.decode(
												    decoder_input_ids[:, :-1] ,       __UpperCamelCase ,       decoder_attention_mask=__UpperCamelCase ,       past_key_values=__UpperCamelCase ,       decoder_position_ids=__UpperCamelCase ,       )

												lowerCamelCase_					=       jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]] ,       dtype="""i4""" )
												lowerCamelCase_					=       model.decode(
												    decoder_input_ids[:, -1:] ,       __UpperCamelCase ,       decoder_attention_mask=__UpperCamelCase ,       past_key_values=outputs_cache.past_key_values ,       decoder_position_ids=__UpperCamelCase ,       )

												lowerCamelCase_					=       model.decode(__UpperCamelCase ,       __UpperCamelCase )

												lowerCamelCase_					=       np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
												self.parent.assertTrue(diff < 1E-3 ,       msg=F'''Max diff is {diff}''' )



					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  List[str] ):
												lowerCamelCase_					=       2_0
												lowerCamelCase_					=       model_class_name(__UpperCamelCase )

												lowerCamelCase_					=       model.encode(inputs_dict["""input_ids"""] )

												lowerCamelCase_	, lowerCamelCase_					=       (
												    inputs_dict["""decoder_input_ids"""],
												    inputs_dict["""decoder_attention_mask"""],
												)

												lowerCamelCase_					=       jnp.concatenate(
												    [
												        decoder_attention_mask,
												        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1]) ),
												    ] ,       axis=-1 ,       )

												lowerCamelCase_					=       model.init_cache(decoder_input_ids.shape[0] ,       __UpperCamelCase ,       __UpperCamelCase )
												lowerCamelCase_					=       jnp.broadcast_to(
												    jnp.arange(decoder_input_ids.shape[-1] - 1 )[None, :] ,       (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1) ,       )

												lowerCamelCase_					=       model.decode(
												    decoder_input_ids[:, :-1] ,       __UpperCamelCase ,       decoder_attention_mask=__UpperCamelCase ,       past_key_values=__UpperCamelCase ,       decoder_position_ids=__UpperCamelCase ,       )
												lowerCamelCase_					=       jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]] ,       dtype="""i4""" )
												lowerCamelCase_					=       model.decode(
												    decoder_input_ids[:, -1:] ,       __UpperCamelCase ,       past_key_values=outputs_cache.past_key_values ,       decoder_attention_mask=__UpperCamelCase ,       decoder_position_ids=__UpperCamelCase ,       )

												lowerCamelCase_					=       model.decode(__UpperCamelCase ,       __UpperCamelCase ,       decoder_attention_mask=__UpperCamelCase )

												lowerCamelCase_					=       np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
												self.parent.assertTrue(diff < 1E-3 ,       msg=F'''Max diff is {diff}''' )





@require_flax
class    __A(					unittest.TestCase							):
					SCREAMING_SNAKE_CASE           =     9_9

					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       np.array(
												    [
												        [7_1, 8_2, 1_8, 3_3, 4_6, 9_1, 2],
												        [6_8, 3_4, 2_6, 5_8, 3_0, 8_2, 2],
												        [5, 9_7, 1_7, 3_9, 9_4, 4_0, 2],
												        [7_6, 8_3, 9_4, 2_5, 7_0, 7_8, 2],
												        [8_7, 5_9, 4_1, 3_5, 4_8, 6_6, 2],
												        [5_5, 1_3, 1_6, 5_8, 5, 2, 1],  # note padding
												        [6_4, 2_7, 3_1, 5_1, 1_2, 7_5, 2],
												        [5_2, 6_4, 8_6, 1_7, 8_3, 3_9, 2],
												        [4_8, 6_1, 9, 2_4, 7_1, 8_2, 2],
												        [2_6, 1, 6_0, 4_8, 2_2, 1_3, 2],
												        [2_1, 5, 6_2, 2_8, 1_4, 7_6, 2],
												        [4_5, 9_8, 3_7, 8_6, 5_9, 4_8, 2],
												        [7_0, 7_0, 5_0, 9, 2_8, 0, 2],
												    ] ,       dtype=np.intaa ,       )

												lowerCamelCase_					=       input_ids.shape[0]
												lowerCamelCase_					=       BlenderbotSmallConfig(
												    vocab_size=self.vocab_size ,       d_model=2_4 ,       encoder_layers=2 ,       decoder_layers=2 ,       encoder_attention_heads=2 ,       decoder_attention_heads=2 ,       encoder_ffn_dim=3_2 ,       decoder_ffn_dim=3_2 ,       max_position_embeddings=4_8 ,       eos_token_id=2 ,       pad_token_id=1 ,       bos_token_id=0 ,       )
												return config, input_ids, batch_size

					def  lowercase__   (						self       :  List[Any] ):
												lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_					=       self._get_config_and_data()
												lowerCamelCase_					=       FlaxBlenderbotSmallForConditionalGeneration(__UpperCamelCase )
												lowerCamelCase_					=       lm_model(input_ids=__UpperCamelCase )
												lowerCamelCase_					=       (batch_size, input_ids.shape[1], config.vocab_size)
												self.assertEqual(outputs["""logits"""].shape ,       __UpperCamelCase )

					def  lowercase__   (						self       :  Tuple ):
												lowerCamelCase_					=       BlenderbotSmallConfig(
												    vocab_size=self.vocab_size ,       d_model=1_4 ,       encoder_layers=2 ,       decoder_layers=2 ,       encoder_attention_heads=2 ,       decoder_attention_heads=2 ,       encoder_ffn_dim=8 ,       decoder_ffn_dim=8 ,       max_position_embeddings=4_8 ,       )
												lowerCamelCase_					=       FlaxBlenderbotSmallForConditionalGeneration(__UpperCamelCase )
												lowerCamelCase_					=       np.array([[7_1, 8_2, 1_8, 3_3, 4_6, 9_1, 2], [6_8, 3_4, 2_6, 5_8, 3_0, 2, 1]] ,       dtype=np.intaa )
												lowerCamelCase_					=       np.array([[8_2, 7_1, 8_2, 1_8, 2], [5_8, 6_8, 2, 1, 1]] ,       dtype=np.intaa )
												lowerCamelCase_					=       lm_model(input_ids=__UpperCamelCase ,       decoder_input_ids=__UpperCamelCase )
												lowerCamelCase_					=       (*summary.shape, config.vocab_size)
												self.assertEqual(outputs["""logits"""].shape ,       __UpperCamelCase )



					def  lowercase__   (						self       :  int ):
												lowerCamelCase_					=       np.array([[7_1, 8_2, 1_8, 3_3, 2, 1, 1], [6_8, 3_4, 2_6, 5_8, 3_0, 8_2, 2]] ,       dtype=np.intaa )
												lowerCamelCase_					=       shift_tokens_right(__UpperCamelCase ,       1 ,       2 )
												lowerCamelCase_					=       np.equal(__UpperCamelCase ,       1 ).astype(np.floataa ).sum()
												lowerCamelCase_					=       np.equal(__UpperCamelCase ,       1 ).astype(np.floataa ).sum()
												self.assertEqual(shifted.shape ,       input_ids.shape )
												self.assertEqual(__UpperCamelCase ,       n_pad_before - 1 )
												self.assertTrue(np.equal(shifted[:, 0] ,       2 ).all() )





@require_flax
class    __A(					UpperCAmelCase  ,		unittest.TestCase  ,		UpperCAmelCase							):
					SCREAMING_SNAKE_CASE           =     True
					SCREAMING_SNAKE_CASE           =     (
					    (
					        FlaxBlenderbotSmallModel,
					        FlaxBlenderbotSmallForConditionalGeneration,
					    )
					    if is_flax_available()
					    else ()
					)
					SCREAMING_SNAKE_CASE           =     (FlaxBlenderbotSmallForConditionalGeneration,) if is_flax_available() else ()

					def  lowercase__   (						self       :  Optional[Any] ):
												lowerCamelCase_					=       FlaxBlenderbotSmallModelTester(self )

					def  lowercase__   (						self       :  str ):
												lowerCamelCase_	, lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												for model_class in self.all_model_classes:
																			self.model_tester.check_use_cache_forward(__UpperCamelCase ,       __UpperCamelCase ,       __UpperCamelCase )

					def  lowercase__   (						self       :  Optional[int] ):
												lowerCamelCase_	, lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												for model_class in self.all_model_classes:
																			self.model_tester.check_use_cache_forward_with_attn_mask(__UpperCamelCase ,       __UpperCamelCase ,       __UpperCamelCase )

					def  lowercase__   (						self       :  List[str] ):
												lowerCamelCase_	, lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																			with self.subTest(model_class.__name__ ):
																										lowerCamelCase_					=       self._prepare_for_class(__UpperCamelCase ,       __UpperCamelCase )
																										lowerCamelCase_					=       model_class(__UpperCamelCase )

																										@jax.jit
																										def encode_jitted(__UpperCamelCase       :  List[Any] ,       __UpperCamelCase       :  Dict=None ,       **__UpperCamelCase       :  Tuple ):
																																	return model.encode(input_ids=__UpperCamelCase ,       attention_mask=__UpperCamelCase )

																										with self.subTest("""JIT Enabled""" ):
																																	lowerCamelCase_					=       encode_jitted(**__UpperCamelCase ).to_tuple()

																										with self.subTest("""JIT Disabled""" ):
																																	with jax.disable_jit():
																																								lowerCamelCase_					=       encode_jitted(**__UpperCamelCase ).to_tuple()

																										self.assertEqual(len(__UpperCamelCase ) ,       len(__UpperCamelCase ) )
																										for jitted_output, output in zip(__UpperCamelCase ,       __UpperCamelCase ):
																																	self.assertEqual(jitted_output.shape ,       output.shape )

					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_	, lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																			with self.subTest(model_class.__name__ ):
																										lowerCamelCase_					=       model_class(__UpperCamelCase )
																										lowerCamelCase_					=       model.encode(inputs_dict["""input_ids"""] ,       inputs_dict["""attention_mask"""] )

																										lowerCamelCase_					=       {
																										    """decoder_input_ids""": inputs_dict["""decoder_input_ids"""],
																										    """decoder_attention_mask""": inputs_dict["""decoder_attention_mask"""],
																										    """encoder_outputs""": encoder_outputs,
																										}

																										@jax.jit
																										def decode_jitted(__UpperCamelCase       :  int ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Union[str, Any] ):
																																	return model.decode(
																																	    decoder_input_ids=__UpperCamelCase ,       decoder_attention_mask=__UpperCamelCase ,       encoder_outputs=__UpperCamelCase ,       )

																										with self.subTest("""JIT Enabled""" ):
																																	lowerCamelCase_					=       decode_jitted(**__UpperCamelCase ).to_tuple()

																										with self.subTest("""JIT Disabled""" ):
																																	with jax.disable_jit():
																																								lowerCamelCase_					=       decode_jitted(**__UpperCamelCase ).to_tuple()

																										self.assertEqual(len(__UpperCamelCase ) ,       len(__UpperCamelCase ) )
																										for jitted_output, output in zip(__UpperCamelCase ,       __UpperCamelCase ):
																																	self.assertEqual(jitted_output.shape ,       output.shape )



					@slow
					def  lowercase__   (						self       :  str ):
												for model_class_name in self.all_model_classes:
																			lowerCamelCase_					=       model_class_name.from_pretrained("""facebook/blenderbot_small-90M""" )
																			# FlaxBlenderbotForSequenceClassification expects eos token in input_ids
																			lowerCamelCase_					=       np.ones((1, 1) ) * model.config.eos_token_id
																			lowerCamelCase_					=       model(__UpperCamelCase )
																			self.assertIsNotNone(__UpperCamelCase )





