



'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


A__    :   Tuple           =					get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')



@require_sentencepiece
@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= SpeechTaTokenizer
 A__				= False
 A__				= True
 def A_   (   self						:							Dict  )      ->   int:







       '''simple docstring'''
       super().setUp()

       # We have a SentencePiece fixture for testing
       __snake_case :   Dict													=    SpeechTaTokenizer(__a  )

       __snake_case :   Any													=    AddedToken('<mask>'      ,     lstrip=__a      ,     rstrip=__a  )
       __snake_case :   List[Any]													=    mask_token
       tokenizer.add_special_tokens({'mask_token': mask_token}  )
       tokenizer.add_tokens(['<ctc_blank>']  )

       tokenizer.save_pretrained(self.tmpdirname  )
 def A_   (   self						:							Optional[Any]      ,     __a						:							int  )      ->   str:







       '''simple docstring'''
       __snake_case :   List[str]													=    'this is a test'
       __snake_case :   List[str]													=    'this is a test'
       return input_text, output_text
 def A_   (   self						:							Any      ,     __a						:							Tuple      ,     __a						:							Optional[int]=False      ,     __a						:							Union[str, Any]=20      ,     __a						:							List[Any]=5  )      ->   Tuple:







       '''simple docstring'''
       __snake_case							,     __snake_case :   str													=    self.get_input_output_texts(__a  )
       __snake_case :   Optional[Any]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Any													=    tokenizer.decode(__a      ,     clean_up_tokenization_spaces=__a  )
       return text, ids
 def A_   (   self						:							Union[str, Any]  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   List[str]													=    '<pad>'
       __snake_case :   str													=    1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a  )      ,     __a  )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    list(self.get_tokenizer().get_vocab().keys()  )

       self.assertEqual(vocab_keys[0]      ,     '<s>'  )
       self.assertEqual(vocab_keys[1]      ,     '<pad>'  )
       self.assertEqual(vocab_keys[-4]      ,     'œ'  )
       self.assertEqual(vocab_keys[-2]      ,     '<mask>'  )
       self.assertEqual(vocab_keys[-1]      ,     '<ctc_blank>'  )
       self.assertEqual(len(__a  )      ,     81  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       self.assertEqual(self.get_tokenizer().vocab_size      ,     79  )
 def A_   (   self						:							str  )      ->   str:







       '''simple docstring'''
       __snake_case :   Optional[int]													=    self.get_tokenizers(do_lower_case=__a  )
       for tokenizer in tokenizers:
             with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
                   __snake_case :   Optional[int]													=    tokenizer.vocab_size
                   __snake_case :   List[Any]													=    len(__a  )

                   self.assertNotEqual(__a      ,     0  )

                   # We usually have added tokens from the start in tests because our vocab fixtures are
                   # smaller than the original vocabs - let's not assert this
                   # self.assertEqual(vocab_size, all_size)

                   __snake_case :   Any													=    ['aaaaa bbbbbb', 'cccccccccdddddddd']
                   __snake_case :   Optional[Any]													=    tokenizer.add_tokens(__a  )
                   __snake_case :   List[str]													=    tokenizer.vocab_size
                   __snake_case :   Dict													=    len(__a  )

                   self.assertNotEqual(__a      ,     0  )
                   self.assertEqual(__a      ,     __a  )
                   self.assertEqual(__a      ,     len(__a  )  )
                   self.assertEqual(__a      ,     all_size + len(__a  )  )

                   __snake_case :   List[Any]													=    tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'      ,     add_special_tokens=__a  )

                   self.assertGreaterEqual(len(__a  )      ,     4  )
                   self.assertGreater(tokens[0]      ,     tokenizer.vocab_size - 1  )
                   self.assertGreater(tokens[-3]      ,     tokenizer.vocab_size - 1  )

                   __snake_case :   Dict													=    {'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
                   __snake_case :   Any													=    tokenizer.add_special_tokens(__a  )
                   __snake_case :   Dict													=    tokenizer.vocab_size
                   __snake_case :   Tuple													=    len(__a  )

                   self.assertNotEqual(__a      ,     0  )
                   self.assertEqual(__a      ,     __a  )
                   self.assertEqual(__a      ,     len(__a  )  )
                   self.assertEqual(__a      ,     all_size_a + len(__a  )  )

                   __snake_case :   List[str]													=    tokenizer.encode(
                       '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'      ,     add_special_tokens=__a  )

                   self.assertGreaterEqual(len(__a  )      ,     6  )
                   self.assertGreater(tokens[0]      ,     tokenizer.vocab_size - 1  )
                   self.assertGreater(tokens[0]      ,     tokens[1]  )
                   self.assertGreater(tokens[-3]      ,     tokenizer.vocab_size - 1  )
                   self.assertGreater(tokens[-3]      ,     tokens[-4]  )
                   self.assertEqual(tokens[0]      ,     tokenizer.eos_token_id  )
                   self.assertEqual(tokens[-3]      ,     tokenizer.pad_token_id  )
 def A_   (   self						:							Optional[int]  )      ->   List[Any]:







       '''simple docstring'''
       pass
 def A_   (   self						:							int  )      ->   Any:







       '''simple docstring'''
       pass
 def A_   (   self						:							int  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   List[Any]													=    self.get_tokenizer()

       __snake_case :   Union[str, Any]													=    tokenizer.tokenize('This is a test'  )
       # fmt: off
       self.assertListEqual(__a      ,     [SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't']  )
       # fmt: on

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__a  )      ,     [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]      ,     )

       __snake_case :   Any													=    tokenizer.tokenize('I was born in 92000, and this is falsé.'  )
       self.assertListEqual(
           __a      ,     [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']  )

       __snake_case :   Any													=    tokenizer.convert_tokens_to_ids(__a  )
       # fmt: off
       self.assertListEqual(__a      ,     [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]  )
       # fmt: on

       __snake_case :   Union[str, Any]													=    tokenizer.convert_ids_to_tokens(__a  )
       self.assertListEqual(
           __a      ,     [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']  )





 @slow
 def A_   (   self						:							Any  )      ->   List[str]:







       '''simple docstring'''
       # Use custom sequence because this tokenizer does not handle numbers.
       __snake_case :   List[Any]													=    [
           'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
           'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
           'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
           'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
           'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
           'conditioning on both left and right context in all layers.',
           'The quick brown fox jumps over the lazy dog.',
       ]

       # fmt: off
       __snake_case :   Dict													=    {
           'input_ids': [
               [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
               [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           ],
           'attention_mask': [
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           ]
       }
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__a      ,     model_name='microsoft/speecht5_asr'      ,     revision='c5ef64c71905caeccde0e4462ef3f9077224c524'      ,     sequences=__a      ,     )








'''simple docstring'''





import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin



class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 A__				= ['''image_processor''', '''tokenizer''']
 A__				= '''AutoImageProcessor'''
 A__				= '''AutoTokenizer'''
 def __init__(   self						:							List[str]      ,     __a						:							Optional[int]=None      ,     __a						:							Any=None      ,     **__a						:							Optional[int]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   str													=    None
       if "feature_extractor" in kwargs:
             warnings.warn(
                 'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
                 ' instead.'      ,     __a      ,     )
             __snake_case :   Tuple													=    kwargs.pop('feature_extractor'  )

       __snake_case :   Union[str, Any]													=    image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
             raise ValueError('You need to specify an `image_processor`.'  )
       if tokenizer is None:
             raise ValueError('You need to specify a `tokenizer`.'  )

       super().__init__(__a      ,     __a  )
       __snake_case :   Any													=    self.image_processor
       __snake_case :   int													=    False
 def __call__(   self						:							str      ,     *__a						:							List[str]      ,     **__a						:							List[str]  )      ->   Dict:







       '''simple docstring'''
       # For backward compatibility
       if self._in_target_context_manager:
             return self.current_processor(*__a      ,     **__a  )

       __snake_case :   List[Any]													=    kwargs.pop('images'      ,     __a  )
       __snake_case :   Any													=    kwargs.pop('text'      ,     __a  )
       if len(__a  ) > 0:
             __snake_case :   List[Any]													=    args[0]
             __snake_case :   Dict													=    args[1:]

       if images is None and text is None:
             raise ValueError('You need to specify either an `images` or `text` input to process.'  )

       if images is not None:
             __snake_case :   int													=    self.image_processor(__a      ,     *__a      ,     **__a  )
       if text is not None:
             __snake_case :   Tuple													=    self.tokenizer(__a      ,     **__a  )

       if text is None:
             return inputs
       elif images is None:
             return encodings
       else:
             __snake_case :   List[str]													=    encodings['input_ids']
             return inputs
 def A_   (   self						:							str      ,     *__a						:							Union[str, Any]      ,     **__a						:							Optional[Any]  )      ->   List[str]:







       '''simple docstring'''
       return self.tokenizer.batch_decode(*__a      ,     **__a  )
 def A_   (   self						:							Any      ,     *__a						:							List[Any]      ,     **__a						:							Dict  )      ->   List[Any]:







       '''simple docstring'''
       return self.tokenizer.decode(*__a      ,     **__a  )
 @contextmanager
 def A_   (   self						:							str  )      ->   List[Any]:







       '''simple docstring'''
       warnings.warn(
           '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
           'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
           'your images inputs, or in a separate call.'  )
       __snake_case :   Union[str, Any]													=    True
       __snake_case :   List[Any]													=    self.tokenizer
       yield
       __snake_case :   str													=    self.image_processor
       __snake_case :   Tuple													=    False
 def A_   (   self						:							List[str]      ,     __a						:							Tuple      ,     __a						:							Union[str, Any]=False      ,     __a						:							str=None  )      ->   List[str]:







       '''simple docstring'''
       if added_vocab is None:
             __snake_case :   List[Any]													=    self.tokenizer.get_added_vocab()

       __snake_case :   Tuple													=    {}

       while tokens:
             __snake_case :   Optional[Any]													=    re.search(r'<s_(.*?)>'      ,     __a      ,     re.IGNORECASE  )
             if start_token is None:
                   break
             __snake_case :   Any													=    start_token.group(1  )
             __snake_case :   Dict													=    re.search(rf'''</s_{key}>'''      ,     __a      ,     re.IGNORECASE  )
             __snake_case :   List[str]													=    start_token.group()
             if end_token is None:
                   __snake_case :   List[Any]													=    tokens.replace(__a      ,     ''  )
             else:
                   __snake_case :   int													=    end_token.group()
                   __snake_case :   Optional[int]													=    re.escape(__a  )
                   __snake_case :   List[str]													=    re.escape(__a  )
                   __snake_case :   Optional[int]													=    re.search(f'''{start_token_escaped}(.*?){end_token_escaped}'''      ,     __a      ,     re.IGNORECASE  )
                   if content is not None:
                         __snake_case :   Tuple													=    content.group(1  ).strip()
                         if r"<s_" in content and r"</s_" in content:  # non-leaf node
                               __snake_case :   str													=    self.tokenajson(__a      ,     is_inner_value=__a      ,     added_vocab=__a  )
                               if value:
                                     if len(__a  ) == 1:
                                           __snake_case :   int													=    value[0]
                                     __snake_case :   Union[str, Any]													=    value
                         else:  # leaf nodes
                               __snake_case :   Any													=    []
                               for leaf in content.split(r'<sep/>'  ):
                                     __snake_case :   str													=    leaf.strip()
                                     if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
                                           __snake_case :   Any													=    leaf[1:-2]  # for categorical special tokens
                                     output[key].append(__a  )
                               if len(output[key]  ) == 1:
                                     __snake_case :   int													=    output[key][0]

                   __snake_case :   Union[str, Any]													=    tokens[tokens.find(__a  ) + len(__a  ) :].strip()
                   if tokens[:6] == r"<sep/>":  # non-leaf nodes
                         return [output] + self.tokenajson(tokens[6:]      ,     is_inner_value=__a      ,     added_vocab=__a  )

       if len(__a  ):
             return [output] if is_inner_value else output
       else:
             return [] if is_inner_value else {"text_sequence": tokens}
 @property
 def A_   (   self						:							Optional[int]  )      ->   Any:







       '''simple docstring'''
       warnings.warn(
           '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'      ,     __a      ,     )
       return self.image_processor_class





 @property
 def A_   (   self						:							Union[str, Any]  )      ->   Optional[int]:







       '''simple docstring'''
       warnings.warn(
           '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'      ,     __a      ,     )
       return self.image_processor



