





"""simple docstring"""


import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__       :					List[str]						=			get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")






@require_sentencepiece
@require_tokenizers
class 							lowercase__			(   __SCREAMING_SNAKE_CASE					,     unittest.TestCase	):

			"""simple docstring"""
			__lowerCAmelCase     :				Tuple										=  SpeechTaTokenizer
			__lowerCAmelCase     :				str										=  False
			__lowerCAmelCase     :				List[str]										=  True







			def   _a      (      self    ):






									'''simple docstring'''







									super().setUp()

									# We have a SentencePiece fixture for testing
									UpperCamelCase				:		List[str]				   =			SpeechTaTokenizer(_A    )

									UpperCamelCase				:		Tuple				   =			AddedToken("""<mask>"""						,   lstrip=_A						,   rstrip=_A    )
									UpperCamelCase				:		Optional[Any]				   =			mask_token
									tokenizer.add_special_tokens({"""mask_token""": mask_token}    )
									tokenizer.add_tokens(["""<ctc_blank>"""]    )

									tokenizer.save_pretrained(self.tmpdirname    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			"""this is a test"""
									UpperCamelCase				:		str				   =			"""this is a test"""
									return input_text, output_text







			def   _a      (      self						,   _A						,   _A=False						,   _A=2_0						,   _A=5    ):






									'''simple docstring'''







									UpperCamelCase   ,						UpperCamelCase				:		List[str]				   =			self.get_input_output_texts(_A    )
									UpperCamelCase				:		Dict				   =			tokenizer.encode(_A						,   add_special_tokens=_A    )
									UpperCamelCase				:		Union[str, Any]				   =			tokenizer.decode(_A						,   clean_up_tokenization_spaces=_A    )
									return text, ids







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			"""<pad>"""
									UpperCamelCase				:		List[Any]				   =			1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A    )						,   _A    )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A    )						,   _A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			list(self.get_tokenizer().get_vocab().keys()    )

									self.assertEqual(vocab_keys[0]						,   """<s>"""    )
									self.assertEqual(vocab_keys[1]						,   """<pad>"""    )
									self.assertEqual(vocab_keys[-4]						,   """œ"""    )
									self.assertEqual(vocab_keys[-2]						,   """<mask>"""    )
									self.assertEqual(vocab_keys[-1]						,   """<ctc_blank>"""    )
									self.assertEqual(len(_A    )						,   8_1    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.assertEqual(self.get_tokenizer().vocab_size						,   7_9    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			self.get_tokenizers(do_lower_case=_A    )
									for tokenizer in tokenizers:
															with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
																					UpperCamelCase				:		Optional[int]				   =			tokenizer.vocab_size
																					UpperCamelCase				:		Union[str, Any]				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )

																					# We usually have added tokens from the start in tests because our vocab fixtures are
																					# smaller than the original vocabs - let's not assert this
																					# self.assertEqual(vocab_size, all_size)

																					UpperCamelCase				:		Dict				   =			["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
																					UpperCamelCase				:		Any				   =			tokenizer.add_tokens(_A    )
																					UpperCamelCase				:		Dict				   =			tokenizer.vocab_size
																					UpperCamelCase				:		int				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )
																					self.assertEqual(_A						,   _A    )
																					self.assertEqual(_A						,   len(_A    )    )
																					self.assertEqual(_A						,   all_size + len(_A    )    )

																					UpperCamelCase				:		Optional[Any]				   =			tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""						,   add_special_tokens=_A    )

																					self.assertGreaterEqual(len(_A    )						,   4    )
																					self.assertGreater(tokens[0]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[-3]						,   tokenizer.vocab_size - 1    )

																					UpperCamelCase				:		int				   =			{"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
																					UpperCamelCase				:		List[Any]				   =			tokenizer.add_special_tokens(_A    )
																					UpperCamelCase				:		str				   =			tokenizer.vocab_size
																					UpperCamelCase				:		Optional[int]				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )
																					self.assertEqual(_A						,   _A    )
																					self.assertEqual(_A						,   len(_A    )    )
																					self.assertEqual(_A						,   all_size_a + len(_A    )    )

																					UpperCamelCase				:		List[Any]				   =			tokenizer.encode(
																					    """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""						,   add_special_tokens=_A    )

																					self.assertGreaterEqual(len(_A    )						,   6    )
																					self.assertGreater(tokens[0]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[0]						,   tokens[1]    )
																					self.assertGreater(tokens[-3]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[-3]						,   tokens[-4]    )
																					self.assertEqual(tokens[0]						,   tokenizer.eos_token_id    )
																					self.assertEqual(tokens[-3]						,   tokenizer.pad_token_id    )







			def   _a      (      self    ):






									'''simple docstring'''







									pass







			def   _a      (      self    ):






									'''simple docstring'''







									pass







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			self.get_tokenizer()

									UpperCamelCase				:		Dict				   =			tokenizer.tokenize("""This is a test"""    )
									# fmt: off
									self.assertListEqual(_A						,   [SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""]    )
									# fmt: on

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_A    )						,   [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]						,   )

									UpperCamelCase				:		Tuple				   =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
									self.assertListEqual(
									    _A						,   [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]    )

									UpperCamelCase				:		Optional[Any]				   =			tokenizer.convert_tokens_to_ids(_A    )
									# fmt: off
									self.assertListEqual(_A						,   [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]    )
									# fmt: on

									UpperCamelCase				:		List[Any]				   =			tokenizer.convert_ids_to_tokens(_A    )
									self.assertListEqual(
									    _A						,   [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]    )







			@slow
			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			[
									    """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
									    """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
									    """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
									    """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
									    """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
									    """conditioning on both left and right context in all layers.""",
									    """The quick brown fox jumps over the lazy dog.""",
									]

									# fmt: off
									UpperCamelCase				:		Optional[Any]				   =			{
									    """input_ids""": [
									        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
									        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									    ],
									    """attention_mask""": [
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									    ]
									}
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=_A						,   model_name="""microsoft/speecht5_asr"""						,   revision="""c5ef64c71905caeccde0e4462ef3f9077224c524"""						,   sequences=_A						,   )











import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


lowerCamelCase      :       int    =						logging.get_logger(__name__)

lowerCamelCase      :       Optional[int]    =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase      :       List[Any]    =						{
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase      :       int    =						{
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase      :       Optional[int]    =						{
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase      :       str    =						{
    '''facebook/dpr-ctx_encoder-single-nq-base''': 512,
    '''facebook/dpr-ctx_encoder-multiset-base''': 512,
}
lowerCamelCase      :       List[str]    =						{
    '''facebook/dpr-question_encoder-single-nq-base''': 512,
    '''facebook/dpr-question_encoder-multiset-base''': 512,
}
lowerCamelCase      :       Union[str, Any]    =						{
    '''facebook/dpr-reader-single-nq-base''': 512,
    '''facebook/dpr-reader-multiset-base''': 512,
}


lowerCamelCase      :       Dict    =						{
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCamelCase      :       Union[str, Any]    =						{
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCamelCase      :       Optional[Any]    =						{
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				DPRContextEncoderTokenizer



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				DPRQuestionEncoderTokenizer

lowerCamelCase      :       Dict    =						collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

lowerCamelCase      :       int    =						collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


lowerCamelCase      :       Optional[int]    =						R'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Return:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''



@add_start_docstrings(a_      )
class 				_UpperCamelCase      :


      def __call__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    __UpperCamelCase = False       ,    __UpperCamelCase = False       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    **__UpperCamelCase       ,    )->						BatchEncoding:
           if titles is None and texts is None:
                return super().__call__(
                    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase       ,    return_attention_mask=__UpperCamelCase       ,    **__UpperCamelCase       ,    )
           elif titles is None or texts is None:
                __lowerCAmelCase             =       titles if texts is None else texts
                return super().__call__(
                    __UpperCamelCase       ,    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase       ,    return_attention_mask=__UpperCamelCase       ,    **__UpperCamelCase       ,    )
           __lowerCAmelCase             =       titles if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [titles]
           __lowerCAmelCase             =       texts if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [texts]
           __lowerCAmelCase             =       len(__UpperCamelCase						)
           __lowerCAmelCase             =       questions if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [questions] * n_passages
           assert len(__UpperCamelCase						) == len(
               __UpperCamelCase						), F"""There should be as many titles than texts but got {len(__UpperCamelCase						)} titles and {len(__UpperCamelCase						)} texts."""
           __lowerCAmelCase             =       super().__call__(__UpperCamelCase       ,    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase						)["input_ids"]
           __lowerCAmelCase             =       super().__call__(__UpperCamelCase       ,    add_special_tokens=__UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase						)["input_ids"]
           __lowerCAmelCase             =       {
               "input_ids": [
                   (encoded_question_and_title + encoded_text)[:max_length]
                   if max_length is not None and truncation
                   else encoded_question_and_title + encoded_text
                   for encoded_question_and_title, encoded_text in zip(__UpperCamelCase       ,    __UpperCamelCase						)
               ]
           }
           if return_attention_mask is not False:
                __lowerCAmelCase             =       []
                for input_ids in encoded_inputs["input_ids"]:
                     attention_mask.append([int(input_id != self.pad_token_id						) for input_id in input_ids]						)
                __lowerCAmelCase             =       attention_mask
           return self.pad(__UpperCamelCase       ,    padding=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase = 1_6       ,    __UpperCamelCase = 6_4       ,    __UpperCamelCase = 4       ,    )->						List[DPRSpanPrediction]:
           __lowerCAmelCase             =       reader_input["input_ids"]
           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       reader_output[:3]
           __lowerCAmelCase             =       len(__UpperCamelCase						)
           __lowerCAmelCase             =       sorted(range(__UpperCamelCase						)       ,    reverse=__UpperCamelCase       ,    key=relevance_logits.__getitem__						)
           __lowerCAmelCase             =       []
           for doc_id in sorted_docs:
                __lowerCAmelCase             =       list(input_ids[doc_id]						)
                # assuming question & title information is at the beginning of the sequence
                __lowerCAmelCase             =       sequence_ids.index(self.sep_token_id       ,    2						) + 1  # second sep id
                if sequence_ids[-1] == self.pad_token_id:
                     __lowerCAmelCase             =       sequence_ids.index(self.pad_token_id						)
                else:
                     __lowerCAmelCase             =       len(__UpperCamelCase						)

                __lowerCAmelCase             =       self._get_best_spans(
                    start_logits=start_logits[doc_id][passage_offset:sequence_len]       ,    end_logits=end_logits[doc_id][passage_offset:sequence_len]       ,    max_answer_length=__UpperCamelCase       ,    top_spans=__UpperCamelCase       ,    )
                for start_index, end_index in best_spans:
                     start_index += passage_offset
                     end_index += passage_offset
                     nbest_spans_predictions.append(
                         DPRSpanPrediction(
                             span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]       ,    relevance_score=relevance_logits[doc_id]       ,    doc_id=__UpperCamelCase       ,    start_index=__UpperCamelCase       ,    end_index=__UpperCamelCase       ,    text=self.decode(sequence_ids[start_index : end_index + 1]						)       ,    )						)
                if len(__UpperCamelCase						) >= num_spans:
                     break
           return nbest_spans_predictions[:num_spans]


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    )->						List[DPRSpanPrediction]:
           __lowerCAmelCase             =       []
           for start_index, start_score in enumerate(__UpperCamelCase						):
                for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]						):
                     scores.append(((start_index, start_index + answer_length), start_score + end_score)						)
           __lowerCAmelCase             =       sorted(__UpperCamelCase       ,    key=lambda __UpperCamelCase						: x[1]       ,    reverse=__UpperCamelCase						)
           __lowerCAmelCase             =       []
           for (start_index, end_index), score in scores:
                assert start_index <= end_index, F"""Wrong span indices: [{start_index}:{end_index}]"""
                __lowerCAmelCase             =       end_index - start_index + 1
                assert length <= max_answer_length, F"""Span is too long: {length} > {max_answer_length}"""
                if any(
                    start_index <= prev_start_index <= prev_end_index <= end_index
                    or prev_start_index <= start_index <= end_index <= prev_end_index
                    for (prev_start_index, prev_end_index) in chosen_span_intervals						):
                     continue
                chosen_span_intervals.append((start_index, end_index)						)

                if len(__UpperCamelCase						) == top_spans:
                     break
           return chosen_span_intervals



@add_end_docstrings(a_      )
class 				_UpperCamelCase      (a_				,		a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				READER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				READER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				["""input_ids""", """attention_mask"""]
      snake_case_	=				DPRReaderTokenizer






