




"""simple docstring"""




import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase				:     Any					=				get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
					from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCAmelCase				:     Optional[Any]					=				5_0003
lowerCAmelCase				:     Any					=				5_0002






@require_sentencepiece
@require_tokenizers
class        __magic_name__				(			UpperCAmelCase__		,   unittest.TestCase							):






				'''simple docstring'''
				__UpperCamelCase         =		PLBartTokenizer
				__UpperCamelCase         =		None
				__UpperCamelCase         =		False





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCamelCase      =   PLBartTokenizer(_a						,					language_codes="""base"""						,					keep_accents=_a					)
							tokenizer.save_pretrained(self.tmpdirname					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   PLBartTokenizer(_a						,					language_codes="""base"""						,					keep_accents=_a					)

							lowerCamelCase      =   tokenizer.tokenize("""This is a test"""					)
							self.assertListEqual(_a						,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_a					)						,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,					)

							lowerCamelCase      =   tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
							self.assertListEqual(
							    _a						,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """é""",
							        """.""",
							    ]						,					)
							lowerCamelCase      =   tokenizer.convert_tokens_to_ids(_a					)
							self.assertListEqual(
							    _a						,					[
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]						,					)

							lowerCamelCase      =   tokenizer.convert_ids_to_tokens(_a					)
							self.assertListEqual(
							    _a						,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """<unk>""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """<unk>""",
							        """.""",
							    ]						,					)

							lowerCamelCase      =   tokenizer.vocab_size
							lowerCamelCase      =   [tokenizer.convert_ids_to_tokens(_a					) for x in range(end - 4						,					_a					)]

							self.assertListEqual(_a						,					["""__java__""", """__python__""", """__en_XX__""", """<mask>"""]					)

							lowerCamelCase      =   """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
							lowerCamelCase      =   tokenizer(_a					).input_ids
							self.assertEqual(
							    tokenizer.decode(_a						,					skip_special_tokens=_a						,					clean_up_tokenization_spaces=_a					)						,					_a						,					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   PLBartTokenizer(_a						,					language_codes="""multi"""						,					keep_accents=_a					)

							lowerCamelCase      =   tokenizer.tokenize("""This is a test"""					)
							self.assertListEqual(_a						,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_a					)						,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,					)

							lowerCamelCase      =   tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
							self.assertListEqual(
							    _a						,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """é""",
							        """.""",
							    ]						,					)
							lowerCamelCase      =   tokenizer.convert_tokens_to_ids(_a					)
							self.assertListEqual(
							    _a						,					[
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]						,					)

							lowerCamelCase      =   tokenizer.convert_ids_to_tokens(_a					)
							self.assertListEqual(
							    _a						,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """<unk>""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """<unk>""",
							        """.""",
							    ]						,					)
							lowerCamelCase      =   tokenizer.vocab_size
							lowerCamelCase      =   [tokenizer.convert_ids_to_tokens(_a					) for x in range(end - 7						,					_a					)]

							self.assertListEqual(
							    _a						,					["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""]					)
							lowerCamelCase      =   """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
							lowerCamelCase      =   tokenizer(_a					).input_ids
							self.assertEqual(
							    tokenizer.decode(_a						,					skip_special_tokens=_a						,					clean_up_tokenization_spaces=_a					)						,					_a						,					)






@require_torch
@require_sentencepiece
@require_tokenizers
class        __magic_name__				(			unittest.TestCase							):






				'''simple docstring'''
				__UpperCamelCase         =		"uclanlp/plbart-python-en_XX"
				__UpperCamelCase         =		[
				    "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
				    "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
				]
				__UpperCamelCase         =		[
				    "Returns the maximum value of a b c.",
				    "Sums the values of a b c.",
				]
				__UpperCamelCase         =		[
				    1_34,
				    54_52,
				    3_34_60,
				    3_34_41,
				    3_34_63,
				    3_34_65,
				    3_34_63,
				    3_34_49,
				    9_88,
				    20,
				    3_34_56,
				    19,
				    3_34_56,
				    7_71,
				    39,
				    42_58,
				    8_89,
				    33_18,
				    3_34_41,
				    3_34_63,
				    3_34_65,
				    3_34_63,
				    3_34_49,
				    24_71,
				    2,
				    PYTHON_CODE,
				]





				@classmethod
				def   _lowerCAmelCase							(			cls					):





							"""simple docstring"""
							lowerCamelCase      =   PLBartTokenizer.from_pretrained(
							    cls.checkpoint_name						,					language_codes="""base"""						,					src_lang="""python"""						,					tgt_lang="""en_XX"""					)
							lowerCamelCase      =   1
							return cls





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""]						,					50_001					)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""]						,					50_002					)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""]						,					50_003					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
							self.assertListEqual(self.expected_src_tokens						,					_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							self.assertIn(_a						,					self.tokenizer.all_special_ids					)
							lowerCamelCase      =   [EN_CODE, 9_037, 33_442, 57, 752, 153, 14, 56, 18, 9, 2]
							lowerCamelCase      =   self.tokenizer.decode(_a						,					skip_special_tokens=_a					)
							lowerCamelCase      =   self.tokenizer.decode(generated_ids[1:]						,					skip_special_tokens=_a					)
							self.assertEqual(_a						,					_a					)
							self.assertNotIn(self.tokenizer.eos_token						,					_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   ["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
							self.assertIsInstance(src_text[0]						,					_a					)
							lowerCamelCase      =   10
							lowerCamelCase      =   self.tokenizer(_a						,					max_length=_a						,					truncation=_a					).input_ids[0]
							self.assertEqual(ids[-2]						,					2					)
							self.assertEqual(ids[-1]						,					_a					)
							self.assertEqual(len(_a					)						,					_a					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""]					)						,					[50_004, 50_001]					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   tempfile.mkdtemp()
							lowerCamelCase      =   self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(_a					)
							lowerCamelCase      =   PLBartTokenizer.from_pretrained(_a					)
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,					_a					)





				@require_torch
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.tokenizer(self.src_text						,					text_target=self.tgt_text						,					padding=_a						,					return_tensors="""pt"""					)
							lowerCamelCase      =   shift_tokens_right(batch["""labels"""]						,					self.tokenizer.pad_token_id					)

							# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
							self.assertEqual(batch.input_ids[1][-2:].tolist()						,					[2, PYTHON_CODE]					)
							self.assertEqual(batch.decoder_input_ids[1][0]						,					_a					)
							self.assertEqual(batch.decoder_input_ids[1][-1]						,					2					)
							self.assertEqual(batch.labels[1][-2:].tolist()						,					[2, EN_CODE]					)





				@require_torch
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.tokenizer(
							    self.src_text						,					text_target=self.tgt_text						,					padding=_a						,					truncation=_a						,					max_length=len(self.expected_src_tokens					)						,					return_tensors="""pt"""						,					)
							lowerCamelCase      =   shift_tokens_right(batch["""labels"""]						,					self.tokenizer.pad_token_id					)

							self.assertIsInstance(_a						,					_a					)

							self.assertEqual((2, 26)						,					batch.input_ids.shape					)
							self.assertEqual((2, 26)						,					batch.attention_mask.shape					)
							lowerCamelCase      =   batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens						,					_a					)
							self.assertEqual(2						,					batch.decoder_input_ids[0, -1]					)  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens						,					[]					)
							self.assertEqual(self.tokenizer.suffix_tokens						,					[self.tokenizer.eos_token_id, PYTHON_CODE]					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.tokenizer(self.src_text						,					padding=_a						,					truncation=_a						,					max_length=3						,					return_tensors="""pt"""					)
							lowerCamelCase      =   self.tokenizer(
							    text_target=self.tgt_text						,					padding=_a						,					truncation=_a						,					max_length=10						,					return_tensors="""pt"""					)
							lowerCamelCase      =   targets["""input_ids"""]
							lowerCamelCase      =   shift_tokens_right(_a						,					self.tokenizer.pad_token_id					)

							self.assertEqual(batch.input_ids.shape[1]						,					3					)
							self.assertEqual(batch.decoder_input_ids.shape[1]						,					10					)





				@require_torch
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   self.tokenizer._build_translation_inputs(
							    """A test"""						,					return_tensors="""pt"""						,					src_lang="""en_XX"""						,					tgt_lang="""java"""					)

							self.assertEqual(
							    nested_simplify(_a					)						,					{
							        # A, test, EOS, en_XX
							        """input_ids""": [[150, 242, 2, 50_003]],
							        """attention_mask""": [[1, 1, 1, 1]],
							        # java
							        """forced_bos_token_id""": 50_001,
							    }						,					)









"""simple docstring"""




from collections.abc import Sequence



def        a__					(							snake_case__			,					snake_case__ = False							)		->   float:
			if not arr:
						return 0

			lowerCamelCase      =   0 if allow_empty_subarrays else float("""-inf"""							)
			lowerCamelCase      =   0.0
			for num in arr:
						lowerCamelCase      =   max(0 if allow_empty_subarrays else num			,					curr_sum + num							)
						lowerCamelCase      =   max(snake_case__			,					snake_case__							)

			return max_sum


if __name__ == "__main__":
					from doctest import testmod

					testmod()

					lowerCAmelCase				:     Union[str, Any]					=				[-2, 1, -3, 4, -1, 2, 1, -5, 4]
					print(F"""{max_subarray_sum(nums) = }""")



