
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__     :		List[str]        = get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")



@require_sentencepiece
@require_tokenizers
class    _snake_case (						UpperCAmelCase_	,    unittest.TestCase				):
 __lowerCAmelCase	:  List[Any]	   =      SpeechTaTokenizer
 __lowerCAmelCase	:  Union[str, Any]	   =      False
 __lowerCAmelCase	:  List[Any]	   =      True


 def   lowercase__  (       self):


   '''simple docstring'''




   super().setUp()

   # We have a SentencePiece fixture for testing
   lowercase__	: int     		=   SpeechTaTokenizer(SCREAMING_SNAKE_CASE_)

   lowercase__	: str     		=   AddedToken("""<mask>"""       ,	lstrip=SCREAMING_SNAKE_CASE_       ,	rstrip=SCREAMING_SNAKE_CASE_)
   lowercase__	: str     		=   mask_token
   tokenizer.add_special_tokens({"""mask_token""": mask_token})
   tokenizer.add_tokens(["""<ctc_blank>"""])

   tokenizer.save_pretrained(self.tmpdirname)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   """this is a test"""
   lowercase__	: List[Any]     		=   """this is a test"""
   return input_text, output_text


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_=False       ,	SCREAMING_SNAKE_CASE_=20       ,	SCREAMING_SNAKE_CASE_=5):


   '''simple docstring'''




   lowercase__		,     lowercase__	: List[Any]     		=   self.get_input_output_texts(SCREAMING_SNAKE_CASE_)
   lowercase__	: int     		=   tokenizer.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
   lowercase__	: Optional[Any]     		=   tokenizer.decode(SCREAMING_SNAKE_CASE_       ,	clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_)
   return text, ids


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   """<pad>"""
   lowercase__	: List[str]     		=   1

   self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_)       ,	SCREAMING_SNAKE_CASE_)
   self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_)       ,	SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: str     		=   list(self.get_tokenizer().get_vocab().keys())

   self.assertEqual(vocab_keys[0]       ,	"""<s>""")
   self.assertEqual(vocab_keys[1]       ,	"""<pad>""")
   self.assertEqual(vocab_keys[-4]       ,	"""œ""")
   self.assertEqual(vocab_keys[-2]       ,	"""<mask>""")
   self.assertEqual(vocab_keys[-1]       ,	"""<ctc_blank>""")
   self.assertEqual(len(SCREAMING_SNAKE_CASE_)       ,	81)


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertEqual(self.get_tokenizer().vocab_size       ,	79)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_)
   for tokenizer in tokenizers:
     with self.subTest(f'{tokenizer.__class__.__name__}'):
       lowercase__	: Dict     		=   tokenizer.vocab_size
       lowercase__	: Dict     		=   len(SCREAMING_SNAKE_CASE_)

       self.assertNotEqual(SCREAMING_SNAKE_CASE_       ,	0)

       # We usually have added tokens from the start in tests because our vocab fixtures are
       # smaller than the original vocabs - let's not assert this
       # self.assertEqual(vocab_size, all_size)

       lowercase__	: Union[str, Any]     		=   ["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
       lowercase__	: int     		=   tokenizer.add_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[int]     		=   tokenizer.vocab_size
       lowercase__	: Any     		=   len(SCREAMING_SNAKE_CASE_)

       self.assertNotEqual(SCREAMING_SNAKE_CASE_       ,	0)
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	len(SCREAMING_SNAKE_CASE_))
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	all_size + len(SCREAMING_SNAKE_CASE_))

       lowercase__	: Dict     		=   tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       self.assertGreaterEqual(len(SCREAMING_SNAKE_CASE_)       ,	4)
       self.assertGreater(tokens[0]       ,	tokenizer.vocab_size - 1)
       self.assertGreater(tokens[-3]       ,	tokenizer.vocab_size - 1)

       lowercase__	: Optional[Any]     		=   {"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
       lowercase__	: Optional[int]     		=   tokenizer.add_special_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: Any     		=   tokenizer.vocab_size
       lowercase__	: Tuple     		=   len(SCREAMING_SNAKE_CASE_)

       self.assertNotEqual(SCREAMING_SNAKE_CASE_       ,	0)
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	len(SCREAMING_SNAKE_CASE_))
       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	all_size_a + len(SCREAMING_SNAKE_CASE_))

       lowercase__	: str     		=   tokenizer.encode(
           """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       self.assertGreaterEqual(len(SCREAMING_SNAKE_CASE_)       ,	6)
       self.assertGreater(tokens[0]       ,	tokenizer.vocab_size - 1)
       self.assertGreater(tokens[0]       ,	tokens[1])
       self.assertGreater(tokens[-3]       ,	tokenizer.vocab_size - 1)
       self.assertGreater(tokens[-3]       ,	tokens[-4])
       self.assertEqual(tokens[0]       ,	tokenizer.eos_token_id)
       self.assertEqual(tokens[-3]       ,	tokenizer.pad_token_id)


 def   lowercase__  (       self):


   '''simple docstring'''




   pass


 def   lowercase__  (       self):


   '''simple docstring'''




   pass


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   self.get_tokenizer()

   lowercase__	: Dict     		=   tokenizer.tokenize("""This is a test""")
   # fmt: off
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	[SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""])
   # fmt: on

   self.assertListEqual(
       tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]       ,	)

   lowercase__	: Optional[Any]     		=   tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
   self.assertListEqual(
       SCREAMING_SNAKE_CASE_       ,	[SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""])

   lowercase__	: Tuple     		=   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)
   # fmt: off
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
   # fmt: on

   lowercase__	: Optional[int]     		=   tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
   self.assertListEqual(
       SCREAMING_SNAKE_CASE_       ,	[SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""])



 @slow
 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   [
       """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
       """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
       """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
       """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
       """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
       """conditioning on both left and right context in all layers.""",
       """The quick brown fox jumps over the lazy dog.""",
   ]

   # fmt: off
   lowercase__	: Dict     		=   {
       """input_ids""": [
           [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
           [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
       ],
       """attention_mask""": [
           [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       ]
   }
   # fmt: on

   self.tokenizer_integration_test_util(
       expected_encoding=SCREAMING_SNAKE_CASE_       ,	model_name="""microsoft/speecht5_asr"""       ,	revision="""c5ef64c71905caeccde0e4462ef3f9077224c524"""       ,	sequences=SCREAMING_SNAKE_CASE_       ,	)








# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


lowerCamelCase__     :		Any        = """Create a default config file for Accelerate with only a few flags set."""

def 		UpperCamelCase						(   lowercase_="no"			,			lowercase_ = default_json_config_file			,			lowercase_ = False )		->						Any:


  '''simple docstring'''






  lowercase__	: Any     		=   Path(lowercase_ )
  path.parent.mkdir(parents=lowercase_			,			exist_ok=lowercase_ )
  if path.exists():
    print(
        F'Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`.' )
    return False
  lowercase__	: int     		=   mixed_precision.lower()
  if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
    raise ValueError(
        F'`mixed_precision` should be one of \'no\', \'fp16\', \'bf16\', or \'fp8\'. Received {mixed_precision}' )
  lowercase__	: Dict     		=   {
      """compute_environment""": """LOCAL_MACHINE""",
      """mixed_precision""": mixed_precision,
  }
  if torch.cuda.is_available():
    lowercase__	: Any     		=   torch.cuda.device_count()
    lowercase__	: Any     		=   num_gpus
    lowercase__	: Optional[int]     		=   False
    if num_gpus > 1:
      lowercase__	: Tuple     		=   """MULTI_GPU"""
    else:
      lowercase__	: Optional[Any]     		=   """NO"""
  elif is_xpu_available() and use_xpu:
    lowercase__	: Union[str, Any]     		=   torch.xpu.device_count()
    lowercase__	: str     		=   num_xpus
    lowercase__	: List[Any]     		=   False
    if num_xpus > 1:
      lowercase__	: str     		=   """MULTI_XPU"""
    else:
      lowercase__	: Optional[Any]     		=   """NO"""
  elif is_npu_available():
    lowercase__	: Tuple     		=   torch.npu.device_count()
    lowercase__	: Union[str, Any]     		=   num_npus
    lowercase__	: Union[str, Any]     		=   False
    if num_npus > 1:
      lowercase__	: List[Any]     		=   """MULTI_NPU"""
    else:
      lowercase__	: int     		=   """NO"""
  else:
    lowercase__	: Union[str, Any]     		=   0
    lowercase__	: str     		=   True
    lowercase__	: Union[str, Any]     		=   1
    lowercase__	: int     		=   """NO"""
  lowercase__	: Tuple     		=   ClusterConfig(**lowercase_ )
  config.to_json_file(lowercase_ )
  return path

def 		UpperCamelCase						(   lowercase_			,			lowercase_ )		->						Optional[Any]:


  '''simple docstring'''






  lowercase__	: List[str]     		=   parser.add_parser("""default"""			,			parents=lowercase_			,			help=lowercase_			,			formatter_class=lowercase_ )
  parser.add_argument(
      """--config_file"""			,			default=lowercase_			,			help=(
          """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
          """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
          """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
          """with 'huggingface'."""
      )			,			dest="""save_location"""			,			)

  parser.add_argument(
      """--mixed_precision"""			,			choices=["""no""", """fp16""", """bf16"""]			,			type=lowercase_			,			help="""Whether or not to use mixed precision training. """
      """Choose between FP16 and BF16 (bfloat16) training. """
      """BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later."""			,			default="""no"""			,			)
  parser.set_defaults(func=lowercase_ )
  return parser



def 		UpperCamelCase						(   lowercase_ )		->						Any:


  '''simple docstring'''






  lowercase__	: Optional[Any]     		=   write_basic_config(args.mixed_precision			,			args.save_location )
  if config_file:
    print(F'accelerate configuration saved at {config_file}' )






