





'''simple docstring'''





import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
      from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
      SCREAMING_SNAKE_CASE_		     = get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

SCREAMING_SNAKE_CASE_		     = 1_2_8_0_2_2
SCREAMING_SNAKE_CASE_		     = 1_2_8_0_2_8

@require_sentencepiece
class 	UpperCamelCase__				(   __UpperCAmelCase							,   unittest.TestCase     ):



     '''simple docstring'''

     __snake_case					:			Dict				=    MaMaaaTokenizer
     __snake_case					:			Any				=    False
     __snake_case					:			Tuple				=    False
     __snake_case					:			List[Any]				=    True






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Union[str, Any]:




           '''simple docstring'''


           super().setUp()

           SCREAMING_SNAKE_CASE	=		["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
           SCREAMING_SNAKE_CASE	=		dict(zip(__SCREAMING_SNAKE_CASE		,range(len(__SCREAMING_SNAKE_CASE ) ) ) )
           SCREAMING_SNAKE_CASE	=		Path(self.tmpdirname )
           save_json(__SCREAMING_SNAKE_CASE		,save_dir / VOCAB_FILES_NAMES["""vocab_file"""] )
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                 copyfile(__SCREAMING_SNAKE_CASE		,save_dir / VOCAB_FILES_NAMES["""spm_file"""] )

           SCREAMING_SNAKE_CASE	=		MaMaaaTokenizer.from_pretrained(self.tmpdirname )
           tokenizer.save_pretrained(self.tmpdirname )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Any		,**lowerCamelCase__		:					str )  ->	Any:




           '''simple docstring'''


           return MaMaaaTokenizer.from_pretrained(self.tmpdirname		,**__SCREAMING_SNAKE_CASE )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int]		,lowerCamelCase__		:					str )  ->	Dict:




           '''simple docstring'''


           return (
               "This is a test",
               "This is a test",
           )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		"""</s>"""
           SCREAMING_SNAKE_CASE	=		0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE )		,__SCREAMING_SNAKE_CASE )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE )		,__SCREAMING_SNAKE_CASE )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.get_tokenizer()
           SCREAMING_SNAKE_CASE	=		list(tokenizer.get_vocab().keys() )

           self.assertEqual(vocab_keys[0]		,"""</s>""" )
           self.assertEqual(vocab_keys[1]		,"""<unk>""" )
           self.assertEqual(vocab_keys[-1]		,"""<s>""" )
           self.assertEqual(len(__SCREAMING_SNAKE_CASE )		,tokenizer.vocab_size + len(tokenizer.get_added_vocab() ) )






     @unittest.skip("""Skip this test while all models are still to be uploaded.""" )
     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Tuple:




           '''simple docstring'''


           pass






     def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.get_tokenizer()

           SCREAMING_SNAKE_CASE	=		tokenizer.tokenize("""This is a test""" )
           self.assertListEqual(__SCREAMING_SNAKE_CASE		,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE )		,[2, 3, 4, 5, 6]		,)

           SCREAMING_SNAKE_CASE	=		tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6] )
           self.assertListEqual(__SCREAMING_SNAKE_CASE		,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

           SCREAMING_SNAKE_CASE	=		tokenizer.convert_tokens_to_string(__SCREAMING_SNAKE_CASE )
           self.assertEqual(__SCREAMING_SNAKE_CASE		,"""This is a test""" )






     @slow
     def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		{"""input_ids""": [[128022, 110108, 397, 11, 38272, 2247, 124811, 285, 18105, 1586, 207, 7, 39534, 4428, 397, 1019, 18105, 1586, 207, 7, 41337, 16786, 241, 7, 20214, 17, 125690, 10398, 7, 44378, 58069, 68342, 7798, 7343, 11, 299, 33310, 4, 158, 37350, 94077, 4569, 299, 33310, 90, 4, 52840, 290, 4, 31270, 112, 299, 682, 4, 52840, 39953, 14079, 193, 52519, 90894, 17894, 120697, 11, 40445, 551, 17, 1019, 52519, 90894, 17756, 963, 11, 40445, 480, 17, 9792, 1120, 5173, 1393, 6240, 16786, 241, 120996, 28, 1245, 1393, 118240, 11123, 1019, 93612, 2691, 10618, 98058, 120409, 1928, 279, 4, 40683, 367, 178, 207, 1019, 103, 103121, 506, 65296, 5, 2], [128022, 21217, 367, 117, 125450, 128, 719, 7, 7308, 40, 93612, 12669, 1116, 16704, 71, 17785, 3699, 15592, 35, 144, 9584, 241, 11943, 713, 950, 799, 2247, 88427, 150, 149, 118813, 120706, 1019, 106906, 81518, 28, 1224, 22799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128022, 1658, 123311, 5155, 5578, 4722, 279, 14947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=__SCREAMING_SNAKE_CASE		,model_name="""facebook/m2m100_418M"""		,revision="""c168bae485c864188cf9aa0e4108b0b6934dc91e"""		,)

@require_torch
@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__				(   unittest.TestCase     ):



     '''simple docstring'''

     __snake_case					:			int				=    '''facebook/m2m100_418M'''
     __snake_case					:			int				=    [
         '''In my opinion, there are two levels of response from the French government.''',
         '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
     ]
     __snake_case					:			List[Any]				=    [
         '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
         '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
     ]

     # fmt: off
     __snake_case					:			List[Any]				=    [EN_CODE, 593, 1949, 115781, 4, 71586, 4234, 60633, 126233, 432, 123808, 15592, 1197, 117132, 120618, 5, 2]






     @classmethod
     def        SCREAMING_SNAKE_CASE__						(							cls		:					Union[str, Any] )  ->	Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		MaMaaaTokenizer.from_pretrained(
               cls.checkpoint_name		,src_lang="""en"""		,tgt_lang="""fr""" )
           SCREAMING_SNAKE_CASE	=		1
           return cls






     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	int:




           '''simple docstring'''


           self.assertEqual(self.tokenizer.get_lang_id("""ar""" )		,128006 )
           self.assertEqual(self.tokenizer.get_lang_id("""en""" )		,128022 )
           self.assertEqual(self.tokenizer.get_lang_id("""ro""" )		,128076 )
           self.assertEqual(self.tokenizer.get_lang_id("""mr""" )		,128063 )






     def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.tokenizer.get_vocab()
           self.assertEqual(len(__SCREAMING_SNAKE_CASE )		,self.tokenizer.vocab_size )
           self.assertEqual(vocab["""<unk>"""]		,3 )
           self.assertIn(self.tokenizer.get_lang_token("""en""" )		,__SCREAMING_SNAKE_CASE )






     def        SCREAMING_SNAKE_CASE__						(							self		:					List[Any] )  ->	str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		"""en"""
           SCREAMING_SNAKE_CASE	=		self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
           self.assertListEqual(self.expected_src_tokens		,__SCREAMING_SNAKE_CASE )






     def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	Union[str, Any]:




           '''simple docstring'''


           self.assertIn(__SCREAMING_SNAKE_CASE		,self.tokenizer.all_special_ids )
           # fmt: off
           SCREAMING_SNAKE_CASE	=		[FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 14028, 136, 3286, 9706, 6, 90797, 6, 144012, 162, 88128, 30061, 5, 2]
           # fmt: on
           SCREAMING_SNAKE_CASE	=		self.tokenizer.decode(__SCREAMING_SNAKE_CASE		,skip_special_tokens=__SCREAMING_SNAKE_CASE )
           SCREAMING_SNAKE_CASE	=		self.tokenizer.decode(generated_ids[1:]		,skip_special_tokens=__SCREAMING_SNAKE_CASE )
           self.assertEqual(__SCREAMING_SNAKE_CASE		,__SCREAMING_SNAKE_CASE )
           self.assertNotIn(self.tokenizer.eos_token		,__SCREAMING_SNAKE_CASE )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		tempfile.mkdtemp()
           SCREAMING_SNAKE_CASE	=		self.tokenizer.lang_token_to_id
           self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE )
           SCREAMING_SNAKE_CASE	=		MaMaaaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE )
           self.assertDictEqual(new_tok.lang_token_to_id		,__SCREAMING_SNAKE_CASE )






     @require_torch
     def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		"""en"""
           SCREAMING_SNAKE_CASE	=		"""fr"""

           SCREAMING_SNAKE_CASE	=		self.tokenizer(self.src_text		,text_target=self.tgt_text		,padding=__SCREAMING_SNAKE_CASE		,return_tensors="""pt""" )

           SCREAMING_SNAKE_CASE	=		shift_tokens_right(
               batch["""labels"""]		,self.tokenizer.pad_token_id		,self.tokenizer.eos_token_id )

           for k in batch:
                 SCREAMING_SNAKE_CASE	=		batch[k].tolist()
           # batch = {k: v.tolist() for k,v in batch.items()}
           # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
           # batch.decoder_inputs_ids[0][0] ==
           assert batch.input_ids[1][0] == EN_CODE
           assert batch.input_ids[1][-1] == 2
           assert batch.labels[1][0] == FR_CODE
           assert batch.labels[1][-1] == 2
           assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]






     @require_torch
     def        SCREAMING_SNAKE_CASE__						(							self		:					Dict )  ->	int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		"""mr"""
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id("""mr""" )] )
           self.assertListEqual(self.tokenizer.suffix_tokens		,[self.tokenizer.eos_token_id] )

           SCREAMING_SNAKE_CASE	=		"""zh"""
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id("""zh""" )] )
           self.assertListEqual(self.tokenizer.suffix_tokens		,[self.tokenizer.eos_token_id] )






     @require_torch
     def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		"""mr"""
           self.tokenizer._switch_to_target_mode()
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id("""mr""" )] )
           self.assertListEqual(self.tokenizer.suffix_tokens		,[self.tokenizer.eos_token_id] )
           self.tokenizer._switch_to_input_mode()
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id(self.tokenizer.src_lang )] )

           SCREAMING_SNAKE_CASE	=		"""zh"""
           self.tokenizer._switch_to_target_mode()
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id("""zh""" )] )
           self.assertListEqual(self.tokenizer.suffix_tokens		,[self.tokenizer.eos_token_id] )
           self.tokenizer._switch_to_input_mode()
           self.assertListEqual(self.tokenizer.prefix_tokens		,[self.tokenizer.get_lang_id(self.tokenizer.src_lang )] )






     @require_torch
     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.tokenizer._build_translation_inputs("""A test"""		,return_tensors="""pt"""		,src_lang="""en"""		,tgt_lang="""ar""" )

           self.assertEqual(
               nested_simplify(__SCREAMING_SNAKE_CASE )		,{
                   # en_XX, A, test, EOS
                   """input_ids""": [[128022, 58, 4183, 2]],
                   """attention_mask""": [[1, 1, 1, 1]],
                   # ar_AR
                   """forced_bos_token_id""": 128006,
               }		,)









import os
from datetime import datetime as dt

from github import Github


SCREAMING_SNAKE_CASE_		     = [
    """good first issue""",
    """feature request""",
    """wip""",
]



def      __lowercase     (   )				->     Optional[int]:


      '''simple docstring'''


      SCREAMING_SNAKE_CASE	=		Github(os.environ["""GITHUB_TOKEN"""]			)
      SCREAMING_SNAKE_CASE	=		g.get_repo("""huggingface/accelerate"""			)
      SCREAMING_SNAKE_CASE	=		repo.get_issues(state="""open"""			)

      for issue in open_issues:
            SCREAMING_SNAKE_CASE	=		sorted([comment for comment in issue.get_comments()]				,		key=lambda _SCREAMING_SNAKE_CASE			: i.created_at				,		reverse=_SCREAMING_SNAKE_CASE			)
            SCREAMING_SNAKE_CASE	=		comments[0] if len(_SCREAMING_SNAKE_CASE			) > 0 else None
            SCREAMING_SNAKE_CASE	=		dt.utcnow()
            SCREAMING_SNAKE_CASE	=		(current_time - issue.updated_at).days
            SCREAMING_SNAKE_CASE	=		(current_time - issue.created_at).days
            if (
                last_comment is not None
                and last_comment.user.login == "github-actions[bot]"
                and days_since_updated > 7
                and days_since_creation >= 30
                and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()			)
            ):
                  # Close issue since it has been 7 days of inactivity since bot mention.
                  issue.edit(state="""closed"""			)
            elif (
                days_since_updated > 23
                and days_since_creation >= 30
                and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()			)
            ):
                  # Add stale comment
                  issue.create_comment(
                      """This issue has been automatically marked as stale because it has not had """
                      """recent activity. If you think this still needs to be addressed """
                      """please comment on this thread.\n\nPlease note that issues that do not follow the """
                      """[contributing guidelines](https://github.com/huggingface/accelerate/blob/main/CONTRIBUTING.md) """
                      """are likely to be ignored."""			)


if __name__ == "__main__":
      main()


