






import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__   :  Tuple        =							get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
       import sentencepiece as sp


__magic_name__   :  Union[str, Any]        =							5
__magic_name__   :  Union[str, Any]        =							10





@require_sentencepiece
@require_tokenizers
class 					A__     (     __snake_case   ,    unittest.TestCase ):

     '''simple docstring'''






     snake_case__					=					SpeechaTextTokenizer
     snake_case__					=					False
     snake_case__					=					True







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           super().setUp()

           UpperCamelCase					       =  sp.SentencePieceProcessor()
           spm_model.Load(_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  ['<s>', '<pad>', '</s>', '<unk>']

           vocab += [spm_model.IdToPiece(id_ ) for id_ in range(len(_SCREAMING_SNAKE_CASE ) )]
           UpperCamelCase					       =  dict(zip(_SCREAMING_SNAKE_CASE   ,						range(len(_SCREAMING_SNAKE_CASE ) ) ) )

           UpperCamelCase					       =  Path(self.tmpdirname )
           save_json(_SCREAMING_SNAKE_CASE   ,						save_dir / VOCAB_FILES_NAMES['vocab_file'] )
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                 copyfile(_SCREAMING_SNAKE_CASE   ,						save_dir / VOCAB_FILES_NAMES['spm_file'] )

           UpperCamelCase					       =  SpeechaTextTokenizer.from_pretrained(self.tmpdirname )
           tokenizer.save_pretrained(self.tmpdirname )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  '<pad>'
           UpperCamelCase					       =  1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  list(self.get_tokenizer().get_vocab().keys() )

           self.assertEqual(vocab_keys[0]   ,						'<s>' )
           self.assertEqual(vocab_keys[1]   ,						'<pad>' )
           self.assertEqual(vocab_keys[-1]   ,						'j' )
           self.assertEqual(len(_SCREAMING_SNAKE_CASE )   ,						1001 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           self.assertEqual(self.get_tokenizer().vocab_size   ,						1001 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  SpeechaTextTokenizer.from_pretrained(self.tmpdirname )

           UpperCamelCase					       =  tokenizer.tokenize('This is a test' )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						['▁This', '▁is', '▁a', '▁t', 'est'] )

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE )   ,						[289, 50, 14, 174, 386]   ,						)

           UpperCamelCase					       =  tokenizer.tokenize('I was born in 92000, and this is falsé.' )
           self.assertListEqual(
               _SCREAMING_SNAKE_CASE   ,						[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']   ,						)
           UpperCamelCase					       =  tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8] )

           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(
               _SCREAMING_SNAKE_CASE   ,						[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']   ,						)







     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  {'input_ids': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=_SCREAMING_SNAKE_CASE   ,						model_name='facebook/s2t-small-mustc-en-de-st'   ,						revision='a14f04cf0776c02f62a8cb800cf7909e15ea23ad'   ,						)





@require_sentencepiece
class 					A__     (     unittest.TestCase ):

     '''simple docstring'''






     snake_case__					=					"""valhalla/s2t_mustc_multilinguial_medium"""

     snake_case__					=					"""C'est trop cool"""
     snake_case__					=					"""Esto es genial"""







     @classmethod
     def   _SCREAMING_SNAKE_CASE     ( cls      :    Tuple ):






           """simple docstring"""




           UpperCamelCase					       =  SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name )
           return cls







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           self.assertEqual(self.tokenizer.lang_code_to_id['pt']   ,						4 )
           self.assertEqual(self.tokenizer.lang_code_to_id['ru']   ,						6 )
           self.assertEqual(self.tokenizer.lang_code_to_id['it']   ,						9 )
           self.assertEqual(self.tokenizer.lang_code_to_id['de']   ,						11 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           self.assertEqual(self.tokenizer.vocab_size   ,						1_0000 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           self.assertIn(_SCREAMING_SNAKE_CASE   ,						self.tokenizer.all_special_ids )
           UpperCamelCase					       =  [ES_CODE, 4, 1601, 47, 7647, 2]
           UpperCamelCase					       =  self.tokenizer.decode(_SCREAMING_SNAKE_CASE   ,						skip_special_tokens=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  self.tokenizer.decode(generated_ids[1:]   ,						skip_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           self.assertNotIn(self.tokenizer.eos_token   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  'fr'
           UpperCamelCase					       =  self.tokenizer(self.french_text ).input_ids
           self.assertEqual(encoded[0]   ,						_SCREAMING_SNAKE_CASE )
           self.assertEqual(encoded[-1]   ,						self.tokenizer.eos_token_id )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Union[str, Any] ):






           """simple docstring"""




           UpperCamelCase					       =  'fr'
           self.assertListEqual(self.tokenizer.prefix_tokens   ,						[FR_CODE] )

           UpperCamelCase					       =  'es'
           self.assertListEqual(self.tokenizer.prefix_tokens   ,						[ES_CODE] )











from datetime import datetime as dt
import os

from github import Github


__magic_name__   :  Dict        =							[
    '''good first issue''',
    '''good second issue''',
    '''good difficult issue''',
    '''feature request''',
    '''new model''',
    '''wip''',
]
def 		lowercase__  (				) ->			Dict:
      """simple docstring"""

      UpperCamelCase					       =  Github(os.environ['GITHUB_TOKEN'])
      UpperCamelCase					       =  g.get_repo('huggingface/transformers')
      UpperCamelCase					       =  repo.get_issues(state='open')

      for issue in open_issues:
            UpperCamelCase					       =  sorted([comment for comment in issue.get_comments()]       ,							key=lambda _UpperCamelCase: i.created_at       ,							reverse=_UpperCamelCase)
            UpperCamelCase					       =  comments[0] if len(_UpperCamelCase) > 0 else None
            if (
                last_comment is not None
                and last_comment.user.login == "github-actions[bot]"
                and (dt.utcnow() - issue.updated_at).days > 7
                and (dt.utcnow() - issue.created_at).days >= 30
                and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels())
            ):
                  # print(f"Would close issue {issue.number} since it has been 7 days of inactivity since bot mention.")
                  issue.edit(state='closed')
            elif (
                (dt.utcnow() - issue.updated_at).days > 23
                and (dt.utcnow() - issue.created_at).days >= 30
                and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels())
            ):
                  # print(f"Would add stale comment to {issue.number}")
                  issue.create_comment(
                      'This issue has been automatically marked as stale because it has not had '
                      'recent activity. If you think this still needs to be addressed '
                      'please comment on this thread.\n\nPlease note that issues that do not follow the '
                      '[contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) '
                      'are likely to be ignored.')


if __name__ == "__main__":
       main()



