





'''simple docstring'''







import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_   :       List[str]       =     get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')
@require_sentencepiece
@require_tokenizers
class 							__SCREAMING_SNAKE_CASE    (__SCREAMING_SNAKE_CASE			,				unittest.TestCase ):






				"""simple docstring"""







				__a  =SpeechTaTokenizer
				__a  =False
				__a  =True




				def     UpperCamelCase__  (    self	:   List[Any]   ):
								super().setUp()

								# We have a SentencePiece fixture for testing
								_a				      =					SpeechTaTokenizer(_A   )

								_a				      =					AddedToken("<mask>"   ,							lstrip=_A   ,							rstrip=_A   )
								_a				      =					mask_token
								tokenizer.add_special_tokens({"mask_token": mask_token}   )
								tokenizer.add_tokens(["<ctc_blank>"]   )

								tokenizer.save_pretrained(self.tmpdirname   )




				def     UpperCamelCase__  (    self	:   List[Any]   ,							__a	:   List[str]   ):
								_a				      =					"""this is a test"""
								_a				      =					"""this is a test"""
								return input_text, output_text




				def     UpperCamelCase__  (    self	:   Optional[Any]   ,							__a	:   List[str]   ,							__a	:   Dict=False   ,							__a	:   Optional[int]=20   ,							__a	:   Dict=5   ):
								_a				      =					self.get_input_output_texts(_A   )
								_a				      =					tokenizer.encode(_A   ,							add_special_tokens=_A   )
								_a				      =					tokenizer.decode(_A   ,							clean_up_tokenization_spaces=_A   )
								return text, ids




				def     UpperCamelCase__  (    self	:   int   ):
								_a				      =					"""<pad>"""
								_a				      =					1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A   )   ,							_A   )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A   )   ,							_A   )




				def     UpperCamelCase__  (    self	:   List[Any]   ):
								_a				      =					list(self.get_tokenizer().get_vocab().keys()   )

								self.assertEqual(vocab_keys[0]   ,							"<s>"   )
								self.assertEqual(vocab_keys[1]   ,							"<pad>"   )
								self.assertEqual(vocab_keys[-4]   ,							"œ"   )
								self.assertEqual(vocab_keys[-2]   ,							"<mask>"   )
								self.assertEqual(vocab_keys[-1]   ,							"<ctc_blank>"   )
								self.assertEqual(len(_A   )   ,							81   )




				def     UpperCamelCase__  (    self	:   Optional[Any]   ):
								self.assertEqual(self.get_tokenizer().vocab_size   ,							79   )




				def     UpperCamelCase__  (    self	:   List[str]   ):
								_a				      =					self.get_tokenizers(do_lower_case=_A   )
								for tokenizer in tokenizers:
												with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																_a				      =					tokenizer.vocab_size
																_a				      =					len(_A   )

																self.assertNotEqual(_A   ,							0   )

																# We usually have added tokens from the start in tests because our vocab fixtures are
																# smaller than the original vocabs - let's not assert this
																# self.assertEqual(vocab_size, all_size)

																_a				      =					["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
																_a				      =					tokenizer.add_tokens(_A   )
																_a				      =					tokenizer.vocab_size
																_a				      =					len(_A   )

																self.assertNotEqual(_A   ,							0   )
																self.assertEqual(_A   ,							_A   )
																self.assertEqual(_A   ,							len(_A   )   )
																self.assertEqual(_A   ,							all_size + len(_A   )   )

																_a				      =					tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l"   ,							add_special_tokens=_A   )

																self.assertGreaterEqual(len(_A   )   ,							4   )
																self.assertGreater(tokens[0]   ,							tokenizer.vocab_size - 1   )
																self.assertGreater(tokens[-3]   ,							tokenizer.vocab_size - 1   )

																_a				      =					{"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
																_a				      =					tokenizer.add_special_tokens(_A   )
																_a				      =					tokenizer.vocab_size
																_a				      =					len(_A   )

																self.assertNotEqual(_A   ,							0   )
																self.assertEqual(_A   ,							_A   )
																self.assertEqual(_A   ,							len(_A   )   )
																self.assertEqual(_A   ,							all_size_a + len(_A   )   )

																_a				      =					tokenizer.encode(
																    ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"   ,							add_special_tokens=_A   )

																self.assertGreaterEqual(len(_A   )   ,							6   )
																self.assertGreater(tokens[0]   ,							tokenizer.vocab_size - 1   )
																self.assertGreater(tokens[0]   ,							tokens[1]   )
																self.assertGreater(tokens[-3]   ,							tokenizer.vocab_size - 1   )
																self.assertGreater(tokens[-3]   ,							tokens[-4]   )
																self.assertEqual(tokens[0]   ,							tokenizer.eos_token_id   )
																self.assertEqual(tokens[-3]   ,							tokenizer.pad_token_id   )




				def     UpperCamelCase__  (    self	:   List[str]   ):
								pass




				def     UpperCamelCase__  (    self	:   Union[str, Any]   ):
								pass




				def     UpperCamelCase__  (    self	:   Tuple   ):
								_a				      =					self.get_tokenizer()

								_a				      =					tokenizer.tokenize("This is a test"   )
								# fmt: off
								self.assertListEqual(_A   ,							[SPIECE_UNDERLINE, "T", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "a", SPIECE_UNDERLINE, "t", "e", "s", "t"]   )
								# fmt: on

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_A   )   ,							[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]   ,							)

								_a				      =					tokenizer.tokenize("I was born in 92000, and this is falsé."   )
								self.assertListEqual(
								    _A   ,							[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "92000", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]   )

								_a				      =					tokenizer.convert_tokens_to_ids(_A   )
								# fmt: off
								self.assertListEqual(_A   ,							[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]   )
								# fmt: on

								_a				      =					tokenizer.convert_ids_to_tokens(_A   )
								self.assertListEqual(
								    _A   ,							[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "<unk>", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]   )




				@slow
				def     UpperCamelCase__  (    self	:   int   ):
								_a				      =					[
								    """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
								    """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
								    """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
								    """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
								    """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
								    """conditioning on both left and right context in all layers.""",
								    """The quick brown fox jumps over the lazy dog.""",
								]

								# fmt: off
								_a				      =					{
								    """input_ids""": [
								        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
								        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
								        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
								    ],
								    """attention_mask""": [
								        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
								        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								    ]
								}
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=_A   ,							model_name="microsoft/speecht5_asr"   ,							revision="c5ef64c71905caeccde0e4462ef3f9077224c524"   ,							sequences=_A   ,							)







"""simple docstring"""


import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						assert isinstance(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""							,     [False, True]     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Tuple				   =			tmp_path / """cache"""
						UpperCamelCase				:		Optional[int]				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCamelCase				:		Tuple				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE							,     keep_in_memory=SCREAMING_SNAKE_CASE     ).read()
						_check_json_dataset(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )

@pytest.mark.parametrize(
    """features"""							,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]							,     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Optional[Any]				   =			tmp_path / """cache"""
						UpperCamelCase				:		int				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						UpperCamelCase				:		Union[str, Any]				   =			features.copy() if features else default_expected_features
						UpperCamelCase				:		str				   =			(
						    Features({feature: Value(SCREAMING_SNAKE_CASE     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						UpperCamelCase				:		Optional[int]				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     features=SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						_check_json_dataset(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )

@pytest.mark.parametrize(
    """features"""							,     [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]							,     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		int				   =			tmp_path / """cache"""
						UpperCamelCase				:		Optional[int]				   =			{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
						UpperCamelCase				:		Optional[Any]				   =			features.copy() if features else default_expected_features
						UpperCamelCase				:		Tuple				   =			(
						    Features({feature: Value(SCREAMING_SNAKE_CASE     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						UpperCamelCase				:		int				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     features=SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						assert isinstance(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
						UpperCamelCase				:		str				   =			{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
						UpperCamelCase				:		Tuple				   =			features.copy()
						UpperCamelCase				:		Tuple				   =			(
						    Features({feature: Value(SCREAMING_SNAKE_CASE     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						UpperCamelCase				:		Optional[int]				   =			tmp_path / """cache"""
						UpperCamelCase				:		List[Any]				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     features=SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						assert isinstance(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""split"""							,     [None, NamedSplit("""train"""     ), """train""", """test"""]     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Optional[int]				   =			tmp_path / """cache"""
						UpperCamelCase				:		Any				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						UpperCamelCase				:		Dict				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE							,     split=SCREAMING_SNAKE_CASE     ).read()
						_check_json_dataset(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						assert dataset.split == split if split else "train"

@pytest.mark.parametrize("""path_type"""							,     [str, list]     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						if issubclass(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
												UpperCamelCase				:		List[str]				   =			jsonl_path
						elif issubclass(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
												UpperCamelCase				:		List[str]				   =			[jsonl_path]
						UpperCamelCase				:		List[Any]				   =			tmp_path / """cache"""
						UpperCamelCase				:		List[Any]				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						UpperCamelCase				:		Optional[int]				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						_check_json_dataset(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE=("train",)     ):
						assert isinstance(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						for split in splits:
												UpperCamelCase				:		Optional[Any]				   =			dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""							,     [False, True]     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		int				   =			tmp_path / """cache"""
						UpperCamelCase				:		int				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCamelCase				:		Any				   =			JsonDatasetReader({"""train""": jsonl_path}							,     cache_dir=SCREAMING_SNAKE_CASE							,     keep_in_memory=SCREAMING_SNAKE_CASE     ).read()
						_check_json_datasetdict(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )

@pytest.mark.parametrize(
    """features"""							,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]							,     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		List[str]				   =			tmp_path / """cache"""
						UpperCamelCase				:		Optional[Any]				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						UpperCamelCase				:		Optional[Any]				   =			features.copy() if features else default_expected_features
						UpperCamelCase				:		Optional[Any]				   =			(
						    Features({feature: Value(SCREAMING_SNAKE_CASE     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						UpperCamelCase				:		Optional[Any]				   =			JsonDatasetReader({"""train""": jsonl_path}							,     features=SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						_check_json_datasetdict(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )

@pytest.mark.parametrize("""split"""							,     [None, NamedSplit("""train"""     ), """train""", """test"""]     )
def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						if split:
												UpperCamelCase				:		List[Any]				   =			{split: jsonl_path}
						else:
												UpperCamelCase				:		Tuple				   =			"""train"""
												UpperCamelCase				:		Union[str, Any]				   =			{"""train""": jsonl_path, """test""": jsonl_path}
						UpperCamelCase				:		Optional[Any]				   =			tmp_path / """cache"""
						UpperCamelCase				:		Optional[int]				   =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						UpperCamelCase				:		Union[str, Any]				   =			JsonDatasetReader(SCREAMING_SNAKE_CASE							,     cache_dir=SCREAMING_SNAKE_CASE     ).read()
						_check_json_datasetdict(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     splits=list(path.keys()     )     )
						assert all(dataset[split].split == split for split in path.keys()     )

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						return json.load(SCREAMING_SNAKE_CASE     )

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						return [json.loads(SCREAMING_SNAKE_CASE     ) for line in buffer]






class 							lowercase__			:

			"""simple docstring"""







			@pytest.mark.parametrize("""lines, load_json_function"""						,   [(True, load_json_lines), (False, load_json)]    )
			def   _a      (      self						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									with io.BytesIO() as buffer:
															JsonDatasetWriter(_A						,   _A						,   lines=_A    ).write()
															buffer.seek(0    )
															UpperCamelCase				:		Optional[Any]				   =			load_json_function(_A    )
									assert isinstance(_A						,   _A    )
									assert isinstance(exported_content[0]						,   _A    )
									assert len(_A    ) == 1_0







			@pytest.mark.parametrize(
			    """orient, container, keys, len_at"""						,   [
			        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
			        ("""split""", dict, {"""columns""", """data"""}, """data"""),
			        ("""index""", dict, set("""0123456789"""    ), None),
			        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
			        ("""values""", list, None, None),
			        ("""table""", dict, {"""schema""", """data"""}, """data"""),
			    ]						,   )
			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									with io.BytesIO() as buffer:
															JsonDatasetWriter(_A						,   _A						,   lines=_A						,   orient=_A    ).write()
															buffer.seek(0    )
															UpperCamelCase				:		str				   =			load_json(_A    )
									assert isinstance(_A						,   _A    )
									if keys:
															if container is dict:
																					assert exported_content.keys() == keys
															else:
																					assert exported_content[0].keys() == keys
									else:
															assert not hasattr(_A						,   """keys"""    ) and not hasattr(exported_content[0]						,   """keys"""    )
									if len_at:
															assert len(exported_content[len_at]    ) == 1_0
									else:
															assert len(_A    ) == 1_0







			@pytest.mark.parametrize("""lines, load_json_function"""						,   [(True, load_json_lines), (False, load_json)]    )
			def   _a      (      self						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									with io.BytesIO() as buffer:
															JsonDatasetWriter(_A						,   _A						,   lines=_A						,   num_proc=2    ).write()
															buffer.seek(0    )
															UpperCamelCase				:		List[Any]				   =			load_json_function(_A    )
									assert isinstance(_A						,   _A    )
									assert isinstance(exported_content[0]						,   _A    )
									assert len(_A    ) == 1_0







			@pytest.mark.parametrize(
			    """orient, container, keys, len_at"""						,   [
			        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
			        ("""split""", dict, {"""columns""", """data"""}, """data"""),
			        ("""index""", dict, set("""0123456789"""    ), None),
			        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
			        ("""values""", list, None, None),
			        ("""table""", dict, {"""schema""", """data"""}, """data"""),
			    ]						,   )
			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									with io.BytesIO() as buffer:
															JsonDatasetWriter(_A						,   _A						,   lines=_A						,   orient=_A						,   num_proc=2    ).write()
															buffer.seek(0    )
															UpperCamelCase				:		Dict				   =			load_json(_A    )
									assert isinstance(_A						,   _A    )
									if keys:
															if container is dict:
																					assert exported_content.keys() == keys
															else:
																					assert exported_content[0].keys() == keys
									else:
															assert not hasattr(_A						,   """keys"""    ) and not hasattr(exported_content[0]						,   """keys"""    )
									if len_at:
															assert len(exported_content[len_at]    ) == 1_0
									else:
															assert len(_A    ) == 1_0







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									with pytest.raises(_A    ):
															with io.BytesIO() as buffer:
																					JsonDatasetWriter(_A						,   _A						,   num_proc=0    )







			@pytest.mark.parametrize("""compression, extension"""						,   [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]    )
			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			tmp_path_factory.mktemp("""data"""    ) / f"""test.json.{extension}"""
									UpperCamelCase				:		Tuple				   =			str(shared_datadir / f"""test_file.json.{extension}"""    )
									JsonDatasetWriter(_A						,   _A						,   compression=_A    ).write()

									with fsspec.open(_A						,   """rb"""						,   compression="""infer"""    ) as f:
															UpperCamelCase				:		Dict				   =			f.read()
									with fsspec.open(_A						,   """rb"""						,   compression="""infer"""    ) as f:
															UpperCamelCase				:		Tuple				   =			f.read()
									assert exported_content == original_content



