
"""simple docstring"""




import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 			lowerCAmelCase_		(_snake_case				,      unittest.TestCase						):


				"""simple docstring"""



				__UpperCamelCase   :		Optional[int]       =     LEDTokenizer
				__UpperCamelCase   :		List[Any]       =     LEDTokenizerFast
				__UpperCamelCase   :		Union[str, Any]       =     True



				def 			__magic_name__					(self					)      ->	Union[str, Any]:



											"""simple docstring"""




											super().setUp()
											SCREAMING_SNAKE_CASE__		: Optional[int]     =     [
											    """l""",
											    """o""",
											    """w""",
											    """e""",
											    """r""",
											    """s""",
											    """t""",
											    """i""",
											    """d""",
											    """n""",
											    """\u0120""",
											    """\u0120l""",
											    """\u0120n""",
											    """\u0120lo""",
											    """\u0120low""",
											    """er""",
											    """\u0120lowest""",
											    """\u0120newer""",
											    """\u0120wider""",
											    """<unk>""",
											]
											SCREAMING_SNAKE_CASE__		: List[Any]     =     dict(zip(lowerCAmelCase__     , range(len(lowerCAmelCase__					)					)					)					)
											SCREAMING_SNAKE_CASE__		: str     =     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
											SCREAMING_SNAKE_CASE__		: str     =     {"""unk_token""": """<unk>"""}

											SCREAMING_SNAKE_CASE__		: str     =     os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["""vocab_file"""]					)
											SCREAMING_SNAKE_CASE__		: Any     =     os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["""merges_file"""]					)
											with open(self.vocab_file     , """w"""     , encoding="""utf-8"""					) as fp:
																		fp.write(json.dumps(lowerCAmelCase__					) + """\n"""					)
											with open(self.merges_file     , """w"""     , encoding="""utf-8"""					) as fp:
																		fp.write("""\n""".join(lowerCAmelCase__					)					)



				def 			__magic_name__					(self     , **SCREAMING_SNAKE_CASE__					)      ->	Optional[int]:



											"""simple docstring"""




											kwargs.update(self.special_tokens_map					)
											return self.tokenizer_class.from_pretrained(self.tmpdirname     , **lowerCAmelCase__					)



				def 			__magic_name__					(self     , **SCREAMING_SNAKE_CASE__					)      ->	Optional[int]:



											"""simple docstring"""




											kwargs.update(self.special_tokens_map					)
											return self.rust_tokenizer_class.from_pretrained(self.tmpdirname     , **lowerCAmelCase__					)



				def 			__magic_name__					(self     , SCREAMING_SNAKE_CASE__					)      ->	List[Any]:



											"""simple docstring"""




											return "lower newer", "lower newer"



				@cached_property
				def 			__magic_name__					(self					)      ->	str:



											"""simple docstring"""




											return LEDTokenizer.from_pretrained("""allenai/led-base-16384"""					)



				@cached_property
				def 			__magic_name__					(self					)      ->	Tuple:



											"""simple docstring"""




											return LEDTokenizerFast.from_pretrained("""allenai/led-base-16384"""					)



				@require_torch
				def 			__magic_name__					(self					)      ->	List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		: Optional[int]     =     ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     [0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: Any     =     tokenizer(lowerCAmelCase__     , max_length=len(lowerCAmelCase__					)     , padding=lowerCAmelCase__     , return_tensors="""pt"""					)
																		self.assertIsInstance(lowerCAmelCase__     , lowerCAmelCase__					)

																		self.assertEqual((2, 9)     , batch.input_ids.shape					)
																		self.assertEqual((2, 9)     , batch.attention_mask.shape					)
																		SCREAMING_SNAKE_CASE__		: Any     =     batch.input_ids.tolist()[0]
																		self.assertListEqual(lowerCAmelCase__     , lowerCAmelCase__					)



				@require_torch
				def 			__magic_name__					(self					)      ->	List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		: Tuple     =     ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: Optional[int]     =     tokenizer(lowerCAmelCase__     , padding=lowerCAmelCase__     , return_tensors="""pt"""					)
																		self.assertIn("""input_ids"""     , lowerCAmelCase__					)
																		self.assertIn("""attention_mask"""     , lowerCAmelCase__					)
																		self.assertNotIn("""labels"""     , lowerCAmelCase__					)
																		self.assertNotIn("""decoder_attention_mask"""     , lowerCAmelCase__					)



				@require_torch
				def 			__magic_name__					(self					)      ->	int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		: Optional[int]     =     [
											    """Summary of the text.""",
											    """Another summary.""",
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: Optional[Any]     =     tokenizer(text_target=lowerCAmelCase__     , max_length=32     , padding="""max_length"""     , return_tensors="""pt"""					)
																		self.assertEqual(32     , targets["""input_ids"""].shape[1]					)



				@require_torch
				def 			__magic_name__					(self					)      ->	int:



											"""simple docstring"""




											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: int     =     tokenizer(
																		    ["""I am a small frog""" * 10_24, """I am a small frog"""]     , padding=lowerCAmelCase__     , truncation=lowerCAmelCase__     , return_tensors="""pt"""					)
																		self.assertIsInstance(lowerCAmelCase__     , lowerCAmelCase__					)
																		self.assertEqual(batch.input_ids.shape     , (2, 51_22)					)



				@require_torch
				def 			__magic_name__					(self					)      ->	Tuple:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		: Tuple     =     ["""A long paragraph for summarization."""]
											SCREAMING_SNAKE_CASE__		: str     =     [
											    """Summary of the text.""",
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: List[str]     =     tokenizer(lowerCAmelCase__     , return_tensors="""pt"""					)
																		SCREAMING_SNAKE_CASE__		: List[Any]     =     tokenizer(text_target=lowerCAmelCase__     , return_tensors="""pt"""					)
																		SCREAMING_SNAKE_CASE__		: Tuple     =     inputs["""input_ids"""]
																		SCREAMING_SNAKE_CASE__		: Any     =     targets["""input_ids"""]
																		self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()					)
																		self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()					)
																		self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()					)
																		self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()					)



				@require_torch
				def 			__magic_name__					(self					)      ->	Optional[Any]:



											"""simple docstring"""




											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		SCREAMING_SNAKE_CASE__		: Optional[Any]     =     ["""Summary of the text.""", """Another summary."""]
																		SCREAMING_SNAKE_CASE__		: Tuple     =     [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

																		SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     tokenizer(lowerCAmelCase__     , padding=lowerCAmelCase__					)
																		SCREAMING_SNAKE_CASE__		: int     =     [[0] * len(lowerCAmelCase__					) for x in encoded_output["""input_ids"""]]
																		SCREAMING_SNAKE_CASE__		: List[Any]     =     tokenizer.pad(lowerCAmelCase__					)
																		self.assertSequenceEqual(outputs["""global_attention_mask"""]     , lowerCAmelCase__					)



				def 			__magic_name__					(self					)      ->	int:



											"""simple docstring"""




											pass



				def 			__magic_name__					(self					)      ->	Union[str, Any]:



											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																									SCREAMING_SNAKE_CASE__		: Optional[int]     =     self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__     , **lowerCAmelCase__					)
																									SCREAMING_SNAKE_CASE__		: Tuple     =     self.tokenizer_class.from_pretrained(lowerCAmelCase__     , **lowerCAmelCase__					)
																									SCREAMING_SNAKE_CASE__		: Any     =     """A, <mask> AllenNLP sentence."""
																									SCREAMING_SNAKE_CASE__		: List[str]     =     tokenizer_r.encode_plus(lowerCAmelCase__     , add_special_tokens=lowerCAmelCase__     , return_token_type_ids=lowerCAmelCase__					)
																									SCREAMING_SNAKE_CASE__		: List[str]     =     tokenizer_p.encode_plus(lowerCAmelCase__     , add_special_tokens=lowerCAmelCase__     , return_token_type_ids=lowerCAmelCase__					)
																									self.assertEqual(sum(tokens_r["""token_type_ids"""]					)     , sum(tokens_p["""token_type_ids"""]					)					)
																									self.assertEqual(
																									    sum(tokens_r["""attention_mask"""]					) / len(tokens_r["""attention_mask"""]					)     , sum(tokens_p["""attention_mask"""]					) / len(tokens_p["""attention_mask"""]					)     , )

																									SCREAMING_SNAKE_CASE__		: Optional[int]     =     tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]					)
																									SCREAMING_SNAKE_CASE__		: Optional[int]     =     tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]					)
																									self.assertSequenceEqual(tokens_p["""input_ids"""]     , [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]					)
																									self.assertSequenceEqual(tokens_r["""input_ids"""]     , [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]					)

																									self.assertSequenceEqual(
																									    lowerCAmelCase__     , ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]					)
																									self.assertSequenceEqual(
																									    lowerCAmelCase__     , ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]					)






"""simple docstring"""




def lowercase_						(     _snake_case     ,_snake_case		):
							return int((input_a, input_a).count(1		) != 0		)

def lowercase_						(     ):
							assert or_gate(0     ,0		) == 0
							assert or_gate(0     ,1		) == 1
							assert or_gate(1     ,0		) == 1
							assert or_gate(1     ,1		) == 1


if __name__ == "__main__":
					print(or_gate(0, 1))
					print(or_gate(1, 0))
					print(or_gate(0, 0))
					print(or_gate(1, 1))




