



"""simple docstring"""
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase      ,  unittest.TestCase   ):
    lowercase__      =   GPTaTokenizer
    lowercase__      =   GPTaTokenizerFast
    lowercase__      =   True
    lowercase__      =   {"add_prefix_space": True}
    lowercase__      =   False

    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      lowercase_								= [
          """l""",
          """o""",
          """w""",
          """e""",
          """r""",
          """s""",
          """t""",
          """i""",
          """d""",
          """n""",
          """\u0120""",
          """\u0120l""",
          """\u0120n""",
          """\u0120lo""",
          """\u0120low""",
          """er""",
          """\u0120lowest""",
          """\u0120newer""",
          """\u0120wider""",
          """<unk>""",
          """<|endoftext|>""",
      ]
      lowercase_								= dict(zip(lowerCAmelCase_ ,  range(len(lowerCAmelCase_))))
      lowercase_								= ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
      lowercase_								= {"""unk_token""": """<unk>"""}

      lowercase_								= os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES["""vocab_file"""])
      lowercase_								= os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES["""merges_file"""])
      with open(self.vocab_file ,  """w""" ,  encoding="""utf-8""") as fp:
        fp.write(json.dumps(lowerCAmelCase_) + """\n""")
      with open(self.merges_file ,  """w""" ,  encoding="""utf-8""") as fp:
        fp.write("""\n""".join(lowerCAmelCase_))

    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  **lowerCAmelCase_			:				Any):







      """simple docstring"""







      kwargs.update(self.special_tokens_map)
      return GPTaTokenizer.from_pretrained(self.tmpdirname ,  **lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				str ,  **lowerCAmelCase_			:				List[str]):







      """simple docstring"""







      kwargs.update(self.special_tokens_map)
      return GPTaTokenizerFast.from_pretrained(self.tmpdirname ,  **lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Optional[int] ,  lowerCAmelCase_			:				Tuple):







      """simple docstring"""







      lowercase_								= """lower newer"""
      lowercase_								= """lower newer"""
      return input_text, output_text

    def 					_UpperCAmelCase	(				self			:				List[Any]):







      """simple docstring"""







      lowercase_								= GPTaTokenizer(self.vocab_file ,  self.merges_file ,  **self.special_tokens_map)
      lowercase_								= """lower newer"""
      lowercase_								= ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
      lowercase_								= tokenizer.tokenize(lowerCAmelCase_ ,  add_prefix_space=lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_ ,  lowerCAmelCase_)

      lowercase_								= tokens + [tokenizer.unk_token]
      lowercase_								= [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_) ,  lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Tuple):







      """simple docstring"""







      if not self.test_rust_tokenizer:
        return

      lowercase_								= self.get_tokenizer()
      lowercase_								= self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_)

      lowercase_								= """lower newer"""

      # Testing tokenization
      lowercase_								= tokenizer.tokenize(lowerCAmelCase_ ,  add_prefix_space=lowerCAmelCase_)
      lowercase_								= rust_tokenizer.tokenize(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_ ,  lowerCAmelCase_)

      # Testing conversion to ids without special tokens
      lowercase_								= tokenizer.encode(lowerCAmelCase_ ,  add_special_tokens=lowerCAmelCase_ ,  add_prefix_space=lowerCAmelCase_)
      lowercase_								= rust_tokenizer.encode(lowerCAmelCase_ ,  add_special_tokens=lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_ ,  lowerCAmelCase_)

      # Testing conversion to ids with special tokens
      lowercase_								= self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_)
      lowercase_								= tokenizer.encode(lowerCAmelCase_ ,  add_prefix_space=lowerCAmelCase_)
      lowercase_								= rust_tokenizer.encode(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_ ,  lowerCAmelCase_)

      # Testing the unknown token
      lowercase_								= tokens + [rust_tokenizer.unk_token]
      lowercase_								= [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_) ,  lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				List[str] ,  *lowerCAmelCase_			:				Tuple ,  **lowerCAmelCase_			:				Any):







      """simple docstring"""







      pass

    def 					_UpperCAmelCase	(				self			:				Dict ,  lowerCAmelCase_			:				Dict=1_5):







      """simple docstring"""







      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
          lowercase_								= self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_ ,  **lowerCAmelCase_)

          # Simple input
          lowercase_								= """This is a simple input"""
          lowercase_								= ["""This is a simple input 1""", """This is a simple input 2"""]
          lowercase_								= ("""This is a simple input""", """This is a pair""")
          lowercase_								= [
              ("""This is a simple input 1""", """This is a simple input 2"""),
              ("""This is a simple pair 1""", """This is a simple pair 2"""),
          ]

          # Simple input tests
          self.assertRaises(lowerCAmelCase_ ,  tokenizer_r.encode ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""")

          # Simple input
          self.assertRaises(lowerCAmelCase_ ,  tokenizer_r.encode_plus ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""")

          # Simple input
          self.assertRaises(
              lowerCAmelCase_ ,  tokenizer_r.batch_encode_plus ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""" ,  )

          # Pair input
          self.assertRaises(lowerCAmelCase_ ,  tokenizer_r.encode ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""")

          # Pair input
          self.assertRaises(lowerCAmelCase_ ,  tokenizer_r.encode_plus ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""")

          # Pair input
          self.assertRaises(
              lowerCAmelCase_ ,  tokenizer_r.batch_encode_plus ,  lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  padding="""max_length""" ,  )

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      lowercase_								= GPTaTokenizer.from_pretrained(self.tmpdirname ,  pad_token="""<pad>""")

      # Simple input
      lowercase_								= """This is a simple input"""
      lowercase_								= ["""This is a simple input looooooooong""", """This is a simple input"""]
      lowercase_								= ("""This is a simple input""", """This is a pair""")
      lowercase_								= [
          ("""This is a simple input loooooong""", """This is a simple input"""),
          ("""This is a simple pair loooooong""", """This is a simple pair"""),
      ]

      lowercase_								= tokenizer.pad_token_id

      lowercase_								= tokenizer(lowerCAmelCase_ ,  padding="""max_length""" ,  max_length=3_0 ,  return_tensors="""np""")
      lowercase_								= tokenizer(lowerCAmelCase_ ,  padding=lowerCAmelCase_ ,  truncate=lowerCAmelCase_ ,  return_tensors="""np""")
      lowercase_								= tokenizer(*lowerCAmelCase_ ,  padding="""max_length""" ,  max_length=6_0 ,  return_tensors="""np""")
      lowercase_								= tokenizer(lowerCAmelCase_ ,  padding=lowerCAmelCase_ ,  truncate=lowerCAmelCase_ ,  return_tensors="""np""")

      # s
      # test single string max_length padding
      self.assertEqual(out_s["""input_ids"""].shape[-1] ,  3_0)
      self.assertTrue(pad_token_id in out_s["""input_ids"""])
      self.assertTrue(0 in out_s["""attention_mask"""])

      # s2
      # test automatic padding
      self.assertEqual(out_sa["""input_ids"""].shape[-1] ,  3_3)
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa["""input_ids"""][0])
      self.assertFalse(0 in out_sa["""attention_mask"""][0])
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa["""input_ids"""][1])
      self.assertTrue(0 in out_sa["""attention_mask"""][1])

      # p
      # test single pair max_length padding
      self.assertEqual(out_p["""input_ids"""].shape[-1] ,  6_0)
      self.assertTrue(pad_token_id in out_p["""input_ids"""])
      self.assertTrue(0 in out_p["""attention_mask"""])

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa["""input_ids"""].shape[-1] ,  5_2)
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa["""input_ids"""][0])
      self.assertFalse(0 in out_pa["""attention_mask"""][0])
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa["""input_ids"""][1])
      self.assertTrue(0 in out_pa["""attention_mask"""][1])

    def 					_UpperCAmelCase	(				self			:				Tuple):







      """simple docstring"""







      lowercase_								= """$$$"""
      lowercase_								= GPTaTokenizer.from_pretrained(self.tmpdirname ,  bos_token=lowerCAmelCase_ ,  add_bos_token=lowerCAmelCase_)

      lowercase_								= """This is a simple input"""
      lowercase_								= ["""This is a simple input 1""", """This is a simple input 2"""]

      lowercase_								= tokenizer.bos_token_id

      lowercase_								= tokenizer(lowerCAmelCase_)
      lowercase_								= tokenizer(lowerCAmelCase_)

      self.assertEqual(out_s.input_ids[0] ,  lowerCAmelCase_)
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

      lowercase_								= tokenizer.decode(out_s.input_ids)
      lowercase_								= tokenizer.batch_decode(out_sa.input_ids)

      self.assertEqual(decode_s.split()[0] ,  lowerCAmelCase_)
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      pass




    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= [self.get_tokenizer(do_lower_case=lowerCAmelCase_ ,  add_bos_token=lowerCAmelCase_)]
      for tokenizer in tokenizers:
        with self.subTest(F'''{tokenizer.__class__.__name__}'''):
          lowercase_								= """Encode this."""
          lowercase_								= """This one too please."""
          lowercase_								= tokenizer.encode(lowerCAmelCase_ ,  add_special_tokens=lowerCAmelCase_)
          encoded_sequence += tokenizer.encode(lowerCAmelCase_ ,  add_special_tokens=lowerCAmelCase_)
          lowercase_								= tokenizer.encode_plus(
              lowerCAmelCase_ ,  lowerCAmelCase_ ,  add_special_tokens=lowerCAmelCase_ ,  return_special_tokens_mask=lowerCAmelCase_ ,  )
          lowercase_								= encoded_sequence_dict["""input_ids"""]
          lowercase_								= encoded_sequence_dict["""special_tokens_mask"""]
          self.assertEqual(len(lowerCAmelCase_) ,  len(lowerCAmelCase_))

          lowercase_								= [
              (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCAmelCase_)
          ]
          lowercase_								= [x for x in filtered_sequence if x is not None]
          self.assertEqual(lowerCAmelCase_ ,  lowerCAmelCase_)







@require_tokenizers
class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):

    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      lowercase_								= AutoTokenizer.from_pretrained("""facebook/opt-350m""" ,  from_slow=lowerCAmelCase_)
      lowercase_								= """A photo of a cat"""

      lowercase_								= tokenizer.encode(
          lowerCAmelCase_ ,  )
      self.assertEqual(lowerCAmelCase_ ,  [2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])
      tokenizer.save_pretrained("""test_opt""")

      lowercase_								= AutoTokenizer.from_pretrained("""./test_opt""")
      lowercase_								= tokenizer.encode(
          lowerCAmelCase_ ,  )
      self.assertEqual(lowerCAmelCase_ ,  [2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      lowercase_								= AutoTokenizer.from_pretrained("""facebook/opt-350m""" ,  use_slow=lowerCAmelCase_)
      lowercase_								= """A photo of a cat"""

      lowercase_								= tokenizer.encode(
          lowerCAmelCase_ ,  )
      # Same as above
      self.assertEqual(lowerCAmelCase_ ,  [2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])




    @unittest.skip("""This test is failing because of a bug in the fast tokenizer""")
    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      lowercase_								= AutoTokenizer.from_pretrained("""facebook/opt-350m""" ,  from_slow=lowerCAmelCase_)

      lowercase_								= """bos"""
      lowercase_								= tokenizer.get_vocab()["""bos"""]

      lowercase_								= """A photo of a cat"""
      lowercase_								= tokenizer.encode(
          lowerCAmelCase_ ,  )
      # We changed the bos token
      self.assertEqual(lowerCAmelCase_ ,  [3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])
      tokenizer.save_pretrained("""./tok""")
      lowercase_								= AutoTokenizer.from_pretrained("""./tok""")
      self.assertTrue(tokenizer.is_fast)
      lowercase_								= tokenizer.encode(
          lowerCAmelCase_ ,  )
      self.assertEqual(lowerCAmelCase_ ,  [3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])






"""simple docstring"""
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase						:		str			= get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
       from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCAmelCase						:		int			= 5_0003
UpperCAmelCase						:		Optional[Any]			= 5_0002


@require_sentencepiece
@require_tokenizers
class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase      ,  unittest.TestCase   ):
    lowercase__      =   PLBartTokenizer
    lowercase__      =   None
    lowercase__      =   False

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      super().setUp()

      # We have a SentencePiece fixture for testing
      lowercase_								= PLBartTokenizer(lowerCAmelCase_ ,  language_codes="""base""" ,  keep_accents=lowerCAmelCase_)
      tokenizer.save_pretrained(self.tmpdirname)

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      lowercase_								= PLBartTokenizer(lowerCAmelCase_ ,  language_codes="""base""" ,  keep_accents=lowerCAmelCase_)

      lowercase_								= tokenizer.tokenize("""This is a test""")
      self.assertListEqual(lowerCAmelCase_ ,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCAmelCase_) ,  [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]] ,  )

      lowercase_								= tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """9""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """é""",
              """.""",
          ] ,  )
      lowercase_								= tokenizer.convert_tokens_to_ids(lowerCAmelCase_)
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
          ] ,  )

      lowercase_								= tokenizer.convert_ids_to_tokens(lowerCAmelCase_)
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """<unk>""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """<unk>""",
              """.""",
          ] ,  )

      lowercase_								= tokenizer.vocab_size
      lowercase_								= [tokenizer.convert_ids_to_tokens(lowerCAmelCase_) for x in range(end - 4 ,  lowerCAmelCase_)]

      self.assertListEqual(lowerCAmelCase_ ,  ["""__java__""", """__python__""", """__en_XX__""", """<mask>"""])

      lowercase_								= """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
      lowercase_								= tokenizer(lowerCAmelCase_).input_ids
      self.assertEqual(
          tokenizer.decode(lowerCAmelCase_ ,  skip_special_tokens=lowerCAmelCase_ ,  clean_up_tokenization_spaces=lowerCAmelCase_) ,  lowerCAmelCase_ ,  )




    def 					_UpperCAmelCase	(				self			:				Optional[Any]):







      """simple docstring"""







      lowercase_								= PLBartTokenizer(lowerCAmelCase_ ,  language_codes="""multi""" ,  keep_accents=lowerCAmelCase_)

      lowercase_								= tokenizer.tokenize("""This is a test""")
      self.assertListEqual(lowerCAmelCase_ ,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCAmelCase_) ,  [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]] ,  )

      lowercase_								= tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """9""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """é""",
              """.""",
          ] ,  )
      lowercase_								= tokenizer.convert_tokens_to_ids(lowerCAmelCase_)
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
          ] ,  )

      lowercase_								= tokenizer.convert_ids_to_tokens(lowerCAmelCase_)
      self.assertListEqual(
          lowerCAmelCase_ ,  [
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """<unk>""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """<unk>""",
              """.""",
          ] ,  )
      lowercase_								= tokenizer.vocab_size
      lowercase_								= [tokenizer.convert_ids_to_tokens(lowerCAmelCase_) for x in range(end - 7 ,  lowerCAmelCase_)]

      self.assertListEqual(
          lowerCAmelCase_ ,  ["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""])
      lowercase_								= """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
      lowercase_								= tokenizer(lowerCAmelCase_).input_ids
      self.assertEqual(
          tokenizer.decode(lowerCAmelCase_ ,  skip_special_tokens=lowerCAmelCase_ ,  clean_up_tokenization_spaces=lowerCAmelCase_) ,  lowerCAmelCase_ ,  )







@require_torch
@require_sentencepiece
@require_tokenizers
class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):
    lowercase__      =   "uclanlp/plbart-python-en_XX"
    lowercase__      =   [
        "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
        "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
    ]
    lowercase__      =   [
        "Returns the maximum value of a b c.",
        "Sums the values of a b c.",
    ]
    lowercase__      =   [
        134,
        5452,
        33460,
        33441,
        33463,
        33465,
        33463,
        33449,
        988,
        20,
        33456,
        19,
        33456,
        771,
        39,
        4258,
        889,
        3318,
        33441,
        33463,
        33465,
        33463,
        33449,
        2471,
        2,
        PYTHON_CODE,
    ]

    @classmethod
    def 					_UpperCAmelCase	(				cls			:				List[Any]):







      """simple docstring"""







      lowercase_								= PLBartTokenizer.from_pretrained(
          cls.checkpoint_name ,  language_codes="""base""" ,  src_lang="""python""" ,  tgt_lang="""en_XX""")
      lowercase_								= 1
      return cls

    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""] ,  5_0_0_0_1)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""] ,  5_0_0_0_2)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""] ,  5_0_0_0_3)

    def 					_UpperCAmelCase	(				self			:				Optional[Any]):







      """simple docstring"""







      lowercase_								= self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
      self.assertListEqual(self.expected_src_tokens ,  lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      self.assertIn(lowerCAmelCase_ ,  self.tokenizer.all_special_ids)
      lowercase_								= [EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
      lowercase_								= self.tokenizer.decode(lowerCAmelCase_ ,  skip_special_tokens=lowerCAmelCase_)
      lowercase_								= self.tokenizer.decode(generated_ids[1:] ,  skip_special_tokens=lowerCAmelCase_)
      self.assertEqual(lowerCAmelCase_ ,  lowerCAmelCase_)
      self.assertNotIn(self.tokenizer.eos_token ,  lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      lowercase_								= ["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 2_0]
      self.assertIsInstance(src_text[0] ,  lowerCAmelCase_)
      lowercase_								= 1_0
      lowercase_								= self.tokenizer(lowerCAmelCase_ ,  max_length=lowerCAmelCase_ ,  truncation=lowerCAmelCase_).input_ids[0]
      self.assertEqual(ids[-2] ,  2)
      self.assertEqual(ids[-1] ,  lowerCAmelCase_)
      self.assertEqual(len(lowerCAmelCase_) ,  lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Optional[Any]):







      """simple docstring"""







      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""]) ,  [5_0_0_0_4, 5_0_0_0_1])

    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= tempfile.mkdtemp()
      lowercase_								= self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(lowerCAmelCase_)
      lowercase_								= PLBartTokenizer.from_pretrained(lowerCAmelCase_)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,  lowerCAmelCase_)

    @require_torch
    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= self.tokenizer(self.src_text ,  text_target=self.tgt_text ,  padding=lowerCAmelCase_ ,  return_tensors="""pt""")
      lowercase_								= shift_tokens_right(batch["""labels"""] ,  self.tokenizer.pad_token_id)

      # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
      self.assertEqual(batch.input_ids[1][-2:].tolist() ,  [2, PYTHON_CODE])
      self.assertEqual(batch.decoder_input_ids[1][0] ,  lowerCAmelCase_)
      self.assertEqual(batch.decoder_input_ids[1][-1] ,  2)
      self.assertEqual(batch.labels[1][-2:].tolist() ,  [2, EN_CODE])

    @require_torch
    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      lowercase_								= self.tokenizer(
          self.src_text ,  text_target=self.tgt_text ,  padding=lowerCAmelCase_ ,  truncation=lowerCAmelCase_ ,  max_length=len(self.expected_src_tokens) ,  return_tensors="""pt""" ,  )
      lowercase_								= shift_tokens_right(batch["""labels"""] ,  self.tokenizer.pad_token_id)

      self.assertIsInstance(lowerCAmelCase_ ,  lowerCAmelCase_)

      self.assertEqual((2, 2_6) ,  batch.input_ids.shape)
      self.assertEqual((2, 2_6) ,  batch.attention_mask.shape)
      lowercase_								= batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens ,  lowerCAmelCase_)
      self.assertEqual(2 ,  batch.decoder_input_ids[0, -1])  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens ,  [])
      self.assertEqual(self.tokenizer.suffix_tokens ,  [self.tokenizer.eos_token_id, PYTHON_CODE])

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= self.tokenizer(self.src_text ,  padding=lowerCAmelCase_ ,  truncation=lowerCAmelCase_ ,  max_length=3 ,  return_tensors="""pt""")
      lowercase_								= self.tokenizer(
          text_target=self.tgt_text ,  padding=lowerCAmelCase_ ,  truncation=lowerCAmelCase_ ,  max_length=1_0 ,  return_tensors="""pt""")
      lowercase_								= targets["""input_ids"""]
      lowercase_								= shift_tokens_right(lowerCAmelCase_ ,  self.tokenizer.pad_token_id)

      self.assertEqual(batch.input_ids.shape[1] ,  3)
      self.assertEqual(batch.decoder_input_ids.shape[1] ,  1_0)




    @require_torch
    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= self.tokenizer._build_translation_inputs(
          """A test""" ,  return_tensors="""pt""" ,  src_lang="""en_XX""" ,  tgt_lang="""java""")

      self.assertEqual(
          nested_simplify(lowerCAmelCase_) ,  {
              # A, test, EOS, en_XX
              """input_ids""": [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
              """attention_mask""": [[1, 1, 1, 1]],
              # java
              """forced_bos_token_id""": 5_0_0_0_1,
          } ,  )

