def 							UpperCamelCase		(			__lowercase		:					bytes      ):
		'''simple docstring'''




		return "".join([hex(__lowercase      )[2:].zfill(2      ).upper() for byte in list(__lowercase      )]      )




def 							UpperCamelCase		(			__lowercase		:					str      ):
		'''simple docstring'''




		if (len(__lowercase      ) % 2) != 0:
				raise ValueError(
				    'Base16 encoded data is invalid:\nData does not have an even number of hex digits.'      )
		# Check the character set - the standard base16 alphabet
		# is uppercase according to RFC3548 section 6
		if not set(__lowercase      ) <= set('0123456789ABCDEF'      ):
				raise ValueError(
				    'Base16 encoded data is invalid:\nData is not uppercase hex or it contains invalid characters.'      )
		# For every two hexadecimal digits (= a byte), turn it into an integer.
		# Then, string the result together into bytes, and return it.
		return bytes(int(data[i] + data[i + 1]		,16      ) for i in range(0		,len(__lowercase      )		,2      )      )


if __name__ == "__main__":
	import doctest

	doctest.testmod()







import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase          =						get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
	from transformers.models.plbart.modeling_plbart import shift_tokens_right

_UpperCAmelCase          =						50003
_UpperCAmelCase          =						50002





@require_sentencepiece
@require_tokenizers
class UpperCAmelCase		(   __A   ,			unittest.TestCase    ):


				'''simple docstring'''






				lowerCamelCase_        =   PLBartTokenizer
				lowerCamelCase_        =   None
				lowerCamelCase_        =   False
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						super().setUp()

						# We have a SentencePiece fixture for testing
						A_						:							str      = PLBartTokenizer(lowercase						,      language_codes='base'						,      keep_accents=lowercase   )
						tokenizer.save_pretrained(self.tmpdirname   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							Tuple      = PLBartTokenizer(lowercase						,      language_codes='base'						,      keep_accents=lowercase   )

						A_						:							Union[str, Any]      = tokenizer.tokenize('This is a test'   )
						self.assertListEqual(lowercase						,      ['▁This', '▁is', '▁a', '▁t', 'est']   )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(lowercase   )						,      [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]						,      )

						A_						:							List[str]      = tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
						self.assertListEqual(
						    lowercase						,      [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]						,      )
						A_						:							List[Any]      = tokenizer.convert_tokens_to_ids(lowercase   )
						self.assertListEqual(
						    lowercase						,      [
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]						,      )

						A_						:							Tuple      = tokenizer.convert_ids_to_tokens(lowercase   )
						self.assertListEqual(
						    lowercase						,      [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '<unk>',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '<unk>',
						        '.',
						    ]						,      )

						A_						:							Optional[Any]      = tokenizer.vocab_size
						A_						:							Dict      = [tokenizer.convert_ids_to_tokens(lowercase   ) for x in range(end - 4						,      lowercase   )]

						self.assertListEqual(lowercase						,      ['__java__', '__python__', '__en_XX__', '<mask>']   )

						A_						:							Dict      = 'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
						A_						:							Optional[Any]      = tokenizer(lowercase   ).input_ids
						self.assertEqual(
						    tokenizer.decode(lowercase						,      skip_special_tokens=lowercase						,      clean_up_tokenization_spaces=lowercase   )						,      lowercase						,      )







				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							str      = PLBartTokenizer(lowercase						,      language_codes='multi'						,      keep_accents=lowercase   )

						A_						:							int      = tokenizer.tokenize('This is a test'   )
						self.assertListEqual(lowercase						,      ['▁This', '▁is', '▁a', '▁t', 'est']   )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(lowercase   )						,      [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]						,      )

						A_						:							List[Any]      = tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
						self.assertListEqual(
						    lowercase						,      [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]						,      )
						A_						:							int      = tokenizer.convert_tokens_to_ids(lowercase   )
						self.assertListEqual(
						    lowercase						,      [
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]						,      )

						A_						:							Tuple      = tokenizer.convert_ids_to_tokens(lowercase   )
						self.assertListEqual(
						    lowercase						,      [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '<unk>',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '<unk>',
						        '.',
						    ]						,      )
						A_						:							Any      = tokenizer.vocab_size
						A_						:							int      = [tokenizer.convert_ids_to_tokens(lowercase   ) for x in range(end - 7						,      lowercase   )]

						self.assertListEqual(
						    lowercase						,      ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']   )
						A_						:							str      = 'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
						A_						:							Dict      = tokenizer(lowercase   ).input_ids
						self.assertEqual(
						    tokenizer.decode(lowercase						,      skip_special_tokens=lowercase						,      clean_up_tokenization_spaces=lowercase   )						,      lowercase						,      )





@require_torch
@require_sentencepiece
@require_tokenizers
class UpperCAmelCase		(   unittest.TestCase    ):


				'''simple docstring'''






				lowerCamelCase_        =   '''uclanlp/plbart-python-en_XX'''
				lowerCamelCase_        =   [
				    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
				    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
				]
				lowerCamelCase_        =   [
				    '''Returns the maximum value of a b c.''',
				    '''Sums the values of a b c.''',
				]
				lowerCamelCase_        =   [
				    1_3_4,
				    5_4_5_2,
				    3_3_4_6_0,
				    3_3_4_4_1,
				    3_3_4_6_3,
				    3_3_4_6_5,
				    3_3_4_6_3,
				    3_3_4_4_9,
				    9_8_8,
				    2_0,
				    3_3_4_5_6,
				    1_9,
				    3_3_4_5_6,
				    7_7_1,
				    3_9,
				    4_2_5_8,
				    8_8_9,
				    3_3_1_8,
				    3_3_4_4_1,
				    3_3_4_6_3,
				    3_3_4_6_5,
				    3_3_4_6_3,
				    3_3_4_4_9,
				    2_4_7_1,
				    2,
				    PYTHON_CODE,
				]
				@classmethod
				def     lowerCAmelCase_  (		cls   ):



						"""simple docstring"""





						A_						:							PLBartTokenizer      = PLBartTokenizer.from_pretrained(
						    cls.checkpoint_name						,      language_codes='base'						,      src_lang='python'						,      tgt_lang='en_XX'   )
						A_						:							Tuple      = 1
						return cls
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']						,      5_0_0_0_1   )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']						,      5_0_0_0_2   )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']						,      5_0_0_0_3   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							int      = self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
						self.assertListEqual(self.expected_src_tokens						,      lowercase   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						self.assertIn(lowercase						,      self.tokenizer.all_special_ids   )
						A_						:							List[Any]      = [EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
						A_						:							Tuple      = self.tokenizer.decode(lowercase						,      skip_special_tokens=lowercase   )
						A_						:							List[str]      = self.tokenizer.decode(generated_ids[1:]						,      skip_special_tokens=lowercase   )
						self.assertEqual(lowercase						,      lowercase   )
						self.assertNotIn(self.tokenizer.eos_token						,      lowercase   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							Any      = ['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 2_0]
						self.assertIsInstance(src_text[0]						,      lowercase   )
						A_						:							int      = 1_0
						A_						:							Dict      = self.tokenizer(lowercase						,      max_length=lowercase						,      truncation=lowercase   ).input_ids[0]
						self.assertEqual(ids[-2]						,      2   )
						self.assertEqual(ids[-1]						,      lowercase   )
						self.assertEqual(len(lowercase   )						,      lowercase   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']   )						,      [5_0_0_0_4, 5_0_0_0_1]   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							str      = tempfile.mkdtemp()
						A_						:							Optional[Any]      = self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(lowercase   )
						A_						:							Union[str, Any]      = PLBartTokenizer.from_pretrained(lowercase   )
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,      lowercase   )
				@require_torch
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							Union[str, Any]      = self.tokenizer(self.src_text						,      text_target=self.tgt_text						,      padding=lowercase						,      return_tensors='pt'   )
						A_						:							List[str]      = shift_tokens_right(batch['labels']						,      self.tokenizer.pad_token_id   )

						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						self.assertEqual(batch.input_ids[1][-2:].tolist()						,      [2, PYTHON_CODE]   )
						self.assertEqual(batch.decoder_input_ids[1][0]						,      lowercase   )
						self.assertEqual(batch.decoder_input_ids[1][-1]						,      2   )
						self.assertEqual(batch.labels[1][-2:].tolist()						,      [2, EN_CODE]   )
				@require_torch
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							Optional[int]      = self.tokenizer(
						    self.src_text						,      text_target=self.tgt_text						,      padding=lowercase						,      truncation=lowercase						,      max_length=len(self.expected_src_tokens   )						,      return_tensors='pt'						,      )
						A_						:							str      = shift_tokens_right(batch['labels']						,      self.tokenizer.pad_token_id   )

						self.assertIsInstance(lowercase						,      lowercase   )

						self.assertEqual((2, 2_6)						,      batch.input_ids.shape   )
						self.assertEqual((2, 2_6)						,      batch.attention_mask.shape   )
						A_						:							Optional[int]      = batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens						,      lowercase   )
						self.assertEqual(2						,      batch.decoder_input_ids[0, -1]   )  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens						,      []   )
						self.assertEqual(self.tokenizer.suffix_tokens						,      [self.tokenizer.eos_token_id, PYTHON_CODE]   )
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							str      = self.tokenizer(self.src_text						,      padding=lowercase						,      truncation=lowercase						,      max_length=3						,      return_tensors='pt'   )
						A_						:							Optional[Any]      = self.tokenizer(
						    text_target=self.tgt_text						,      padding=lowercase						,      truncation=lowercase						,      max_length=1_0						,      return_tensors='pt'   )
						A_						:							Optional[int]      = targets['input_ids']
						A_						:							int      = shift_tokens_right(lowercase						,      self.tokenizer.pad_token_id   )

						self.assertEqual(batch.input_ids.shape[1]						,      3   )
						self.assertEqual(batch.decoder_input_ids.shape[1]						,      1_0   )







				@require_torch
				def     lowerCAmelCase_  (		self   ):



						"""simple docstring"""





						A_						:							List[str]      = self.tokenizer._build_translation_inputs(
						    'A test'						,      return_tensors='pt'						,      src_lang='en_XX'						,      tgt_lang='java'   )

						self.assertEqual(
						    nested_simplify(lowercase   )						,      {
						        # A, test, EOS, en_XX
						        'input_ids': [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
						        'attention_mask': [[1, 1, 1, 1]],
						        # java
						        'forced_bos_token_id': 5_0_0_0_1,
						    }						,      )






