





import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__      =get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
       from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCamelCase__      =5_0003
UpperCamelCase__      =5_0002





@require_sentencepiece
@require_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    PLBartTokenizer
       __snake_case 			=    None
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     List[str]:
            super().setUp()

            # We have a SentencePiece fixture for testing
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							PLBartTokenizer(__lowerCamelCase    ,						language_codes="base"    ,						keep_accents=__lowerCamelCase							)
            tokenizer.save_pretrained(self.tmpdirname							)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							PLBartTokenizer(__lowerCamelCase    ,						language_codes="base"    ,						keep_accents=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer.tokenize("This is a test"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁This", "▁is", "▁a", "▁t", "est"]							)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,						)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							tokenizer.tokenize("I was born in 92000, and this is falsé."							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    SPIECE_UNDERLINE + "I",
                    SPIECE_UNDERLINE + "was",
                    SPIECE_UNDERLINE + "b",
                    "or",
                    "n",
                    SPIECE_UNDERLINE + "in",
                    SPIECE_UNDERLINE + "",
                    "9",
                    "2",
                    "0",
                    "0",
                    "0",
                    ",",
                    SPIECE_UNDERLINE + "and",
                    SPIECE_UNDERLINE + "this",
                    SPIECE_UNDERLINE + "is",
                    SPIECE_UNDERLINE + "f",
                    "al",
                    "s",
                    "é",
                    ".",
                ]    ,						)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer.convert_tokens_to_ids(__lowerCamelCase							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    value + tokenizer.fairseq_offset
                    for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                ]    ,						)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.convert_ids_to_tokens(__lowerCamelCase							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    SPIECE_UNDERLINE + "I",
                    SPIECE_UNDERLINE + "was",
                    SPIECE_UNDERLINE + "b",
                    "or",
                    "n",
                    SPIECE_UNDERLINE + "in",
                    SPIECE_UNDERLINE + "",
                    "<unk>",
                    "2",
                    "0",
                    "0",
                    "0",
                    ",",
                    SPIECE_UNDERLINE + "and",
                    SPIECE_UNDERLINE + "this",
                    SPIECE_UNDERLINE + "is",
                    SPIECE_UNDERLINE + "f",
                    "al",
                    "s",
                    "<unk>",
                    ".",
                ]    ,						)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tokenizer.vocab_size
            _SCREAMING_SNAKE_CASE       :	Any						    =							[tokenizer.convert_ids_to_tokens(__lowerCamelCase							) for x in range(end - 4    ,						__lowerCamelCase							)]

            self.assertListEqual(__lowerCamelCase    ,						["__java__", "__python__", "__en_XX__", "<mask>"]							)

            _SCREAMING_SNAKE_CASE       :	str						    =							"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer(__lowerCamelCase							).input_ids
            self.assertEqual(
                tokenizer.decode(__lowerCamelCase    ,						skip_special_tokens=__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)    ,						__lowerCamelCase    ,						)







       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							PLBartTokenizer(__lowerCamelCase    ,						language_codes="multi"    ,						keep_accents=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.tokenize("This is a test"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁This", "▁is", "▁a", "▁t", "est"]							)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,						)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize("I was born in 92000, and this is falsé."							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    SPIECE_UNDERLINE + "I",
                    SPIECE_UNDERLINE + "was",
                    SPIECE_UNDERLINE + "b",
                    "or",
                    "n",
                    SPIECE_UNDERLINE + "in",
                    SPIECE_UNDERLINE + "",
                    "9",
                    "2",
                    "0",
                    "0",
                    "0",
                    ",",
                    SPIECE_UNDERLINE + "and",
                    SPIECE_UNDERLINE + "this",
                    SPIECE_UNDERLINE + "is",
                    SPIECE_UNDERLINE + "f",
                    "al",
                    "s",
                    "é",
                    ".",
                ]    ,						)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.convert_tokens_to_ids(__lowerCamelCase							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    value + tokenizer.fairseq_offset
                    for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                ]    ,						)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.convert_ids_to_tokens(__lowerCamelCase							)
            self.assertListEqual(
                __lowerCamelCase    ,						[
                    SPIECE_UNDERLINE + "I",
                    SPIECE_UNDERLINE + "was",
                    SPIECE_UNDERLINE + "b",
                    "or",
                    "n",
                    SPIECE_UNDERLINE + "in",
                    SPIECE_UNDERLINE + "",
                    "<unk>",
                    "2",
                    "0",
                    "0",
                    "0",
                    ",",
                    SPIECE_UNDERLINE + "and",
                    SPIECE_UNDERLINE + "this",
                    SPIECE_UNDERLINE + "is",
                    SPIECE_UNDERLINE + "f",
                    "al",
                    "s",
                    "<unk>",
                    ".",
                ]    ,						)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.vocab_size
            _SCREAMING_SNAKE_CASE       :	int						    =							[tokenizer.convert_ids_to_tokens(__lowerCamelCase							) for x in range(end - 7    ,						__lowerCamelCase							)]

            self.assertListEqual(
                __lowerCamelCase    ,						["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"]							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer(__lowerCamelCase							).input_ids
            self.assertEqual(
                tokenizer.decode(__lowerCamelCase    ,						skip_special_tokens=__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)    ,						__lowerCamelCase    ,						)





@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    'uclanlp/plbart-python-en_XX'
       __snake_case 			=    [
           'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
           'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
       ]
       __snake_case 			=    [
           'Returns the maximum value of a b c.',
           'Sums the values of a b c.',
       ]
       __snake_case 			=    [
           1_3_4,
           5_4_5_2,
           3_3_4_6_0,
           3_3_4_4_1,
           3_3_4_6_3,
           3_3_4_6_5,
           3_3_4_6_3,
           3_3_4_4_9,
           9_8_8,
           2_0,
           3_3_4_5_6,
           1_9,
           3_3_4_5_6,
           7_7_1,
           3_9,
           4_2_5_8,
           8_8_9,
           3_3_1_8,
           3_3_4_4_1,
           3_3_4_6_3,
           3_3_4_6_5,
           3_3_4_6_3,
           3_3_4_4_9,
           2_4_7_1,
           2,
           PYTHON_CODE,
       ]




       @classmethod
       def     UpperCamelCase_			(      cls							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	PLBartTokenizer						    =							PLBartTokenizer.from_pretrained(
                cls.checkpoint_name    ,						language_codes="base"    ,						src_lang="python"    ,						tgt_lang="en_XX"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							1
            return cls




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]    ,						5_0_0_0_1							)
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]    ,						5_0_0_0_2							)
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]    ,						5_0_0_0_3							)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
            self.assertListEqual(self.expected_src_tokens    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Tuple:
            self.assertIn(__lowerCamelCase    ,						self.tokenizer.all_special_ids							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							[EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.tokenizer.decode(__lowerCamelCase    ,						skip_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							self.tokenizer.decode(generated_ids[1:]    ,						skip_special_tokens=__lowerCamelCase							)
            self.assertEqual(__lowerCamelCase    ,						__lowerCamelCase							)
            self.assertNotIn(self.tokenizer.eos_token    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	int						    =							["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 2_0]
            self.assertIsInstance(src_text[0]    ,						__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							1_0
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.tokenizer(__lowerCamelCase    ,						max_length=__lowerCamelCase    ,						truncation=__lowerCamelCase							).input_ids[0]
            self.assertEqual(ids[-2]    ,						2							)
            self.assertEqual(ids[-1]    ,						__lowerCamelCase							)
            self.assertEqual(len(__lowerCamelCase							)    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"]							)    ,						[5_0_0_0_4, 5_0_0_0_1]							)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tempfile.mkdtemp()
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer.fairseq_tokens_to_ids
            self.tokenizer.save_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							PLBartTokenizer.from_pretrained(__lowerCamelCase							)
            self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,						__lowerCamelCase							)




       @require_torch
       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.tokenizer(self.src_text    ,						text_target=self.tgt_text    ,						padding=__lowerCamelCase    ,						return_tensors="pt"							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							shift_tokens_right(batch["labels"]    ,						self.tokenizer.pad_token_id							)

            # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
            self.assertEqual(batch.input_ids[1][-2:].tolist()    ,						[2, PYTHON_CODE]							)
            self.assertEqual(batch.decoder_input_ids[1][0]    ,						__lowerCamelCase							)
            self.assertEqual(batch.decoder_input_ids[1][-1]    ,						2							)
            self.assertEqual(batch.labels[1][-2:].tolist()    ,						[2, EN_CODE]							)




       @require_torch
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer(
                self.src_text    ,						text_target=self.tgt_text    ,						padding=__lowerCamelCase    ,						truncation=__lowerCamelCase    ,						max_length=len(self.expected_src_tokens							)    ,						return_tensors="pt"    ,						)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							shift_tokens_right(batch["labels"]    ,						self.tokenizer.pad_token_id							)

            self.assertIsInstance(__lowerCamelCase    ,						__lowerCamelCase							)

            self.assertEqual((2, 2_6)    ,						batch.input_ids.shape							)
            self.assertEqual((2, 2_6)    ,						batch.attention_mask.shape							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							batch.input_ids.tolist()[0]
            self.assertListEqual(self.expected_src_tokens    ,						__lowerCamelCase							)
            self.assertEqual(2    ,						batch.decoder_input_ids[0, -1]							)  # EOS
            # Test that special tokens are reset
            self.assertEqual(self.tokenizer.prefix_tokens    ,						[]							)
            self.assertEqual(self.tokenizer.suffix_tokens    ,						[self.tokenizer.eos_token_id, PYTHON_CODE]							)




       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer(self.src_text    ,						padding=__lowerCamelCase    ,						truncation=__lowerCamelCase    ,						max_length=3    ,						return_tensors="pt"							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.tokenizer(
                text_target=self.tgt_text    ,						padding=__lowerCamelCase    ,						truncation=__lowerCamelCase    ,						max_length=1_0    ,						return_tensors="pt"							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							targets["input_ids"]
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							shift_tokens_right(__lowerCamelCase    ,						self.tokenizer.pad_token_id							)

            self.assertEqual(batch.input_ids.shape[1]    ,						3							)
            self.assertEqual(batch.decoder_input_ids.shape[1]    ,						1_0							)







       @require_torch
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.tokenizer._build_translation_inputs(
                "A test"    ,						return_tensors="pt"    ,						src_lang="en_XX"    ,						tgt_lang="java"							)

            self.assertEqual(
                nested_simplify(__lowerCamelCase							)    ,						{
                    # A, test, EOS, en_XX
                    "input_ids": [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
                    "attention_mask": [[1, 1, 1, 1]],
                    # java
                    "forced_bos_token_id": 5_0_0_0_1,
                }    ,						)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     while a != 0:
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Any						    =							b % a, a
     return b






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     if gcd(__lowerCamelCase,       __lowerCamelCase	) != 1:
          _SCREAMING_SNAKE_CASE       :	int						    =							f"""mod inverse of {a!r} and {m!r} does not exist"""
          raise ValueError(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	List[str]						    =							1, 0, a
     _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	List[str]						    =							0, 1, m
     while va != 0:
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							ua // va
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							(ua - q * va), (ua - q * va), (ua - q * va), va, va, va
     return ua % m