






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__							=     get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
     from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase__							=     5_0003
lowerCamelCase__							=     5_0002




@require_sentencepiece
@require_tokenizers
class 			snake_case__   (							lowercase_	,    unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= PLBartTokenizer
       lowerCamelCase							:				Tuple 				= None
       lowerCamelCase							:				Dict 				= False



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            super().setUp()

            # We have a SentencePiece fixture for testing
            __snake_case			:int          =						PLBartTokenizer(a__			,						language_codes="""base"""			,						keep_accents=a__    )
            tokenizer.save_pretrained(self.tmpdirname    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						PLBartTokenizer(a__			,						language_codes="""base"""			,						keep_accents=a__    )

            __snake_case			:str          =						tokenizer.tokenize("""This is a test"""    )
            self.assertListEqual(a__			,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(a__    )			,						[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]			,						)

            __snake_case			:Union[str, Any]          =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]			,						)
            __snake_case			:Union[str, Any]          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(
                a__			,						[
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                ]			,						)

            __snake_case			:int          =						tokenizer.convert_ids_to_tokens(a__    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]			,						)

            __snake_case			:int          =						tokenizer.vocab_size
            __snake_case			:List[Any]          =						[tokenizer.convert_ids_to_tokens(a__    ) for x in range(end - 4			,						a__    )]

            self.assertListEqual(a__			,						["""__java__""", """__python__""", """__en_XX__""", """<mask>"""]    )

            __snake_case			:Optional[Any]          =						"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
            __snake_case			:Union[str, Any]          =						tokenizer(a__    ).input_ids
            self.assertEqual(
                tokenizer.decode(a__			,						skip_special_tokens=a__			,						clean_up_tokenization_spaces=a__    )			,						a__			,						)



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						PLBartTokenizer(a__			,						language_codes="""multi"""			,						keep_accents=a__    )

            __snake_case			:str          =						tokenizer.tokenize("""This is a test"""    )
            self.assertListEqual(a__			,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(a__    )			,						[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]			,						)

            __snake_case			:Optional[int]          =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]			,						)
            __snake_case			:List[str]          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(
                a__			,						[
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                ]			,						)

            __snake_case			:str          =						tokenizer.convert_ids_to_tokens(a__    )
            self.assertListEqual(
                a__			,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]			,						)
            __snake_case			:int          =						tokenizer.vocab_size
            __snake_case			:str          =						[tokenizer.convert_ids_to_tokens(a__    ) for x in range(end - 7			,						a__    )]

            self.assertListEqual(
                a__			,						["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""]    )
            __snake_case			:List[str]          =						"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
            __snake_case			:str          =						tokenizer(a__    ).input_ids
            self.assertEqual(
                tokenizer.decode(a__			,						skip_special_tokens=a__			,						clean_up_tokenization_spaces=a__    )			,						a__			,						)




@require_torch
@require_sentencepiece
@require_tokenizers
class 			snake_case__   (							unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= "uclanlp/plbart-python-en_XX"
       lowerCamelCase							:				Tuple 				= [
           "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
           "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
       ]
       lowerCamelCase							:				Any 				= [
           "Returns the maximum value of a b c.",
           "Sums the values of a b c.",
       ]
       lowerCamelCase							:				Tuple 				= [
           134,
           5452,
           3_3460,
           3_3441,
           3_3463,
           3_3465,
           3_3463,
           3_3449,
           988,
           20,
           3_3456,
           19,
           3_3456,
           771,
           39,
           4258,
           889,
           3318,
           3_3441,
           3_3463,
           3_3465,
           3_3463,
           3_3449,
           2471,
           2,
           PYTHON_CODE,
       ]



       @classmethod
       def 					__lowercase						(							cls    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:PLBartTokenizer          =						PLBartTokenizer.from_pretrained(
                cls.checkpoint_name			,						language_codes="""base"""			,						src_lang="""python"""			,						tgt_lang="""en_XX"""    )
            __snake_case			:Any          =						1
            return cls



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""]			,						5_00_01    )
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""]			,						5_00_02    )
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""]			,						5_00_03    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:Any          =						self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
            self.assertListEqual(self.expected_src_tokens			,						a__    )



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            self.assertIn(a__			,						self.tokenizer.all_special_ids    )
            __snake_case			:Dict          =						[EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
            __snake_case			:List[Any]          =						self.tokenizer.decode(a__			,						skip_special_tokens=a__    )
            __snake_case			:Union[str, Any]          =						self.tokenizer.decode(generated_ids[1:]			,						skip_special_tokens=a__    )
            self.assertEqual(a__			,						a__    )
            self.assertNotIn(self.tokenizer.eos_token			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
            self.assertIsInstance(src_text[0]			,						a__    )
            __snake_case			:Optional[Any]          =						10
            __snake_case			:Tuple          =						self.tokenizer(a__			,						max_length=a__			,						truncation=a__    ).input_ids[0]
            self.assertEqual(ids[-2]			,						2    )
            self.assertEqual(ids[-1]			,						a__    )
            self.assertEqual(len(a__    )			,						a__    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""]    )			,						[5_00_04, 5_00_01]    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						tempfile.mkdtemp()
            __snake_case			:List[str]          =						self.tokenizer.fairseq_tokens_to_ids
            self.tokenizer.save_pretrained(a__    )
            __snake_case			:List[str]          =						PLBartTokenizer.from_pretrained(a__    )
            self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,						a__    )



       @require_torch
       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:str          =						self.tokenizer(self.src_text			,						text_target=self.tgt_text			,						padding=a__			,						return_tensors="""pt"""    )
            __snake_case			:Any          =						shift_tokens_right(batch["""labels"""]			,						self.tokenizer.pad_token_id    )

            # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
            self.assertEqual(batch.input_ids[1][-2:].tolist()			,						[2, PYTHON_CODE]    )
            self.assertEqual(batch.decoder_input_ids[1][0]			,						a__    )
            self.assertEqual(batch.decoder_input_ids[1][-1]			,						2    )
            self.assertEqual(batch.labels[1][-2:].tolist()			,						[2, EN_CODE]    )



       @require_torch
       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.tokenizer(
                self.src_text			,						text_target=self.tgt_text			,						padding=a__			,						truncation=a__			,						max_length=len(self.expected_src_tokens    )			,						return_tensors="""pt"""			,						)
            __snake_case			:List[Any]          =						shift_tokens_right(batch["""labels"""]			,						self.tokenizer.pad_token_id    )

            self.assertIsInstance(a__			,						a__    )

            self.assertEqual((2, 26)			,						batch.input_ids.shape    )
            self.assertEqual((2, 26)			,						batch.attention_mask.shape    )
            __snake_case			:str          =						batch.input_ids.tolist()[0]
            self.assertListEqual(self.expected_src_tokens			,						a__    )
            self.assertEqual(2			,						batch.decoder_input_ids[0, -1]    )  # EOS
            # Test that special tokens are reset
            self.assertEqual(self.tokenizer.prefix_tokens			,						[]    )
            self.assertEqual(self.tokenizer.suffix_tokens			,						[self.tokenizer.eos_token_id, PYTHON_CODE]    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tokenizer(self.src_text			,						padding=a__			,						truncation=a__			,						max_length=3			,						return_tensors="""pt"""    )
            __snake_case			:int          =						self.tokenizer(
                text_target=self.tgt_text			,						padding=a__			,						truncation=a__			,						max_length=10			,						return_tensors="""pt"""    )
            __snake_case			:Dict          =						targets["""input_ids"""]
            __snake_case			:Union[str, Any]          =						shift_tokens_right(a__			,						self.tokenizer.pad_token_id    )

            self.assertEqual(batch.input_ids.shape[1]			,						3    )
            self.assertEqual(batch.decoder_input_ids.shape[1]			,						10    )



       @require_torch
       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case			:Any          =						self.tokenizer._build_translation_inputs(
                """A test"""			,						return_tensors="""pt"""			,						src_lang="""en_XX"""			,						tgt_lang="""java"""    )

            self.assertEqual(
                nested_simplify(a__    )			,						{
                    # A, test, EOS, en_XX
                    """input_ids""": [[1_50, 2_42, 2, 5_00_03]],
                    """attention_mask""": [[1, 1, 1, 1]],
                    # java
                    """forced_bos_token_id""": 5_00_01,
                }			,						)










import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


lowerCamelCase__							=     """
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
"""




class 			snake_case__   (							unittest.TestCase	,    lowercase_):

       '''simple docstring'''



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:int          =						load_tool("""text-question-answering"""    )
            self.tool.setup()
            __snake_case			:Tuple          =						load_tool("""text-question-answering"""			,						remote=a__    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tool(a__			,						"""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.remote_tool(a__			,						"""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.tool(text=a__			,						question="""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.remote_tool(text=a__			,						question="""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )


