





import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase    : Union[str, Any]    	=  get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
       import sentencepiece as sp


lowercase    : List[str]    	=  5
lowercase    : List[str]    	=  10

@require_sentencepiece
@require_tokenizers
class  __A(						__UpperCAmelCase						,			unittest.TestCase ):
     __A	  =       SpeechaTextTokenizer
     __A	  =       False
     __A	  =       True

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            super().setUp()

            _UpperCamelCase              =						sp.SentencePieceProcessor()
            spm_model.Load(A      )
            _UpperCamelCase              =						['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

            vocab += [spm_model.IdToPiece(id_      ) for id_ in range(len(A      )      )]
            _UpperCamelCase              =						dict(zip(A,       range(len(A      )      )      )      )

            _UpperCamelCase              =						Path(self.tmpdirname      )
            save_json(A,       save_dir / VOCAB_FILES_NAMES['''vocab_file''']      )
            if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                   copyfile(A,       save_dir / VOCAB_FILES_NAMES['''spm_file''']      )

            _UpperCamelCase              =						SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
            tokenizer.save_pretrained(self.tmpdirname      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''<pad>'''
            _UpperCamelCase              =						1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(A      ),       A      )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(A      ),       A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						list(self.get_tokenizer().get_vocab().keys()      )

            self.assertEqual(vocab_keys[0],       '''<s>'''      )
            self.assertEqual(vocab_keys[1],       '''<pad>'''      )
            self.assertEqual(vocab_keys[-1],       '''j'''      )
            self.assertEqual(len(A      ),       1001      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(self.get_tokenizer().vocab_size,       1001      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )

            _UpperCamelCase              =						tokenizer.tokenize('''This is a test'''      )
            self.assertListEqual(A,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(A      ),       [289, 50, 14, 174, 386],       )

            _UpperCamelCase              =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
            self.assertListEqual(
                A,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],       )
            _UpperCamelCase              =						tokenizer.convert_tokens_to_ids(A      )
            self.assertListEqual(A,       [12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]      )

            _UpperCamelCase              =						tokenizer.convert_ids_to_tokens(A      )
            self.assertListEqual(
                A,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],       )



     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						{'''input_ids''': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=A,       model_name='''facebook/s2t-small-mustc-en-de-st''',       revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',       )






@require_sentencepiece
class  __A(						unittest.TestCase ):
     __A	  =       "valhalla/s2t_mustc_multilinguial_medium"

     __A	  =       "C'est trop cool"
     __A	  =       "Esto es genial"

     @classmethod
     def 	_UpperCamelCase     (      cls      ):





            """simple docstring"""

            _UpperCamelCase              =						SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name      )
            return cls

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],       4      )
            self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],       6      )
            self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],       9      )
            self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],       11      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(self.tokenizer.vocab_size,       1_0000      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertIn(A,       self.tokenizer.all_special_ids      )
            _UpperCamelCase              =						[ES_CODE, 4, 1601, 47, 7647, 2]
            _UpperCamelCase              =						self.tokenizer.decode(A,       skip_special_tokens=A      )
            _UpperCamelCase              =						self.tokenizer.decode(generated_ids[1:],       skip_special_tokens=A      )
            self.assertEqual(A,       A      )
            self.assertNotIn(self.tokenizer.eos_token,       A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''fr'''
            _UpperCamelCase              =						self.tokenizer(self.french_text      ).input_ids
            self.assertEqual(encoded[0],       A      )
            self.assertEqual(encoded[-1],       self.tokenizer.eos_token_id      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''fr'''
            self.assertListEqual(self.tokenizer.prefix_tokens,       [FR_CODE]      )

            _UpperCamelCase              =						'''es'''
            self.assertListEqual(self.tokenizer.prefix_tokens,       [ES_CODE]      )













import os
import sys
import unittest


lowercase    : Any    	=  os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
lowercase    : int    	=  os.path.join(git_repo_path, """src""", """transformers""")

lowercase    : str    	=  """
{0} = None
"""

lowercase    : Union[str, Any]    	=  """
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})
"""


lowercase    : Optional[Any]    	=  """
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
"""

class  __A(						unittest.TestCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						find_backend('''    _import_structure["models.albert"].append("AlbertTokenizerFast")'''      )
            self.assertIsNone(A      )

            _UpperCamelCase              =						find_backend('''    if not is_tokenizers_available():'''      )
            self.assertEqual(A,       '''tokenizers'''      )

            _UpperCamelCase              =						find_backend('''    if not is_tensorflow_text_available():'''      )
            self.assertEqual(A,       '''tensorflow_text'''      )

            _UpperCamelCase              =						find_backend('''    if not (is_sentencepiece_available() and is_tokenizers_available()):'''      )
            self.assertEqual(A,       '''sentencepiece_and_tokenizers'''      )

            _UpperCamelCase              =						find_backend(
                '''    if not (is_sentencepiece_available() and is_tensorflow_text_available()):'''      )
            self.assertEqual(A,       '''sentencepiece_and_tensorflow_text'''      )

            _UpperCamelCase              =						find_backend(
                '''    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):'''      )
            self.assertEqual(A,       '''sentencepiece_and_tokenizers_and_vision'''      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						read_init()
            # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
            self.assertIn('''torch''',       A      )
            self.assertIn('''tensorflow_text''',       A      )
            self.assertIn('''sentencepiece_and_tokenizers''',       A      )

            # Likewise, we can't assert on the exact content of a key
            self.assertIn('''BertModel''',       objects['''torch''']      )
            self.assertIn('''TFBertModel''',       objects['''tf''']      )
            self.assertIn('''FlaxBertModel''',       objects['''flax''']      )
            self.assertIn('''BertModel''',       objects['''torch''']      )
            self.assertIn('''TFBertTokenizer''',       objects['''tensorflow_text''']      )
            self.assertIn('''convert_slow_tokenizer''',       objects['''sentencepiece_and_tokenizers''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						create_dummy_object('''CONSTANT''',       '''\'torch\''''      )
            self.assertEqual(A,       '''\nCONSTANT = None\n'''      )

            _UpperCamelCase              =						create_dummy_object('''function''',       '''\'torch\''''      )
            self.assertEqual(
                A,       '''\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n'''      )

            _UpperCamelCase              =						'''
class FakeClass(metaclass=DummyObject):
    _backends = \'torch\'

    def __init__(self, *args, **kwargs):
        requires_backends(self, \'torch\')
'''
            _UpperCamelCase              =						create_dummy_object('''FakeClass''',       '''\'torch\''''      )
            self.assertEqual(A,       A      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, ["torch"])


class FakeClass(metaclass=DummyObject):
    _backends = ["torch"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, ["torch"])
'''
            _UpperCamelCase              =						create_dummy_files({'''torch''': ['''CONSTANT''', '''function''', '''FakeClass''']}      )
            self.assertEqual(dummy_files['''torch'''],       A      )






