


'''simple docstring'''



import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							__UpperCamelCase    (		_A		,				unittest.TestCase ):
	A_           =   LEDTokenizer
	A_           =   LEDTokenizerFast
	A_           =   True




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()
						__a      :	int    =  [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						]
						__a      :	Any    =  dict(zip(__SCREAMING_SNAKE_CASE						,    range(len(__SCREAMING_SNAKE_CASE							)							)							)							)
						__a      :	str    =  ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						__a      :	Union[str, Any]    =  {'unk_token': '<unk>'}

						__a      :	Optional[int]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['vocab_file']							)
						__a      :	int    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['merges_file']							)
						with open(self.vocab_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write(json.dumps(__SCREAMING_SNAKE_CASE							) + '\n'							)
						with open(self.merges_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write('\n'.join(__SCREAMING_SNAKE_CASE							)							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return self.tokenizer_class.from_pretrained(self.tmpdirname						,    **__SCREAMING_SNAKE_CASE							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return self.rust_tokenizer_class.from_pretrained(self.tmpdirname						,    **__SCREAMING_SNAKE_CASE							)




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						return "lower newer", "lower newer"




	@cached_property
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						return LEDTokenizer.from_pretrained('allenai/led-base-16384'							)




	@cached_property
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Union[str, Any]    =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
						__a      :	Optional[Any]    =  [0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	Union[str, Any]    =  tokenizer(__SCREAMING_SNAKE_CASE						,    max_length=len(__SCREAMING_SNAKE_CASE							)						,    padding=__SCREAMING_SNAKE_CASE						,    return_tensors='pt'							)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE						,    __SCREAMING_SNAKE_CASE							)

											self.assertEqual((2, 9)						,    batch.input_ids.shape							)
											self.assertEqual((2, 9)						,    batch.attention_mask.shape							)
											__a      :	Optional[int]    =  batch.input_ids.tolist()[0]
											self.assertListEqual(__SCREAMING_SNAKE_CASE						,    __SCREAMING_SNAKE_CASE							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[Any]    =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	List[str]    =  tokenizer(__SCREAMING_SNAKE_CASE						,    padding=__SCREAMING_SNAKE_CASE						,    return_tensors='pt'							)
											self.assertIn('input_ids'						,    __SCREAMING_SNAKE_CASE							)
											self.assertIn('attention_mask'						,    __SCREAMING_SNAKE_CASE							)
											self.assertNotIn('labels'						,    __SCREAMING_SNAKE_CASE							)
											self.assertNotIn('decoder_attention_mask'						,    __SCREAMING_SNAKE_CASE							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  [
						    'Summary of the text.',
						    'Another summary.',
						]
						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	Tuple    =  tokenizer(text_target=__SCREAMING_SNAKE_CASE						,    max_length=32						,    padding='max_length'						,    return_tensors='pt'							)
											self.assertEqual(32						,    targets['input_ids'].shape[1]							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	Tuple    =  tokenizer(
											    ['I am a small frog' * 1024, 'I am a small frog']						,    padding=__SCREAMING_SNAKE_CASE						,    truncation=__SCREAMING_SNAKE_CASE						,    return_tensors='pt'							)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE						,    __SCREAMING_SNAKE_CASE							)
											self.assertEqual(batch.input_ids.shape						,    (2, 5122)							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  ['A long paragraph for summarization.']
						__a      :	List[Any]    =  [
						    'Summary of the text.',
						]
						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	Any    =  tokenizer(__SCREAMING_SNAKE_CASE						,    return_tensors='pt'							)
											__a      :	int    =  tokenizer(text_target=__SCREAMING_SNAKE_CASE						,    return_tensors='pt'							)
											__a      :	Dict    =  inputs['input_ids']
											__a      :	Any    =  targets['input_ids']
											self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()							)
											self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()							)
											self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()							)
											self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
											__a      :	Optional[Any]    =  ['Summary of the text.', 'Another summary.']
											__a      :	List[str]    =  [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

											__a      :	List[str]    =  tokenizer(__SCREAMING_SNAKE_CASE						,    padding=__SCREAMING_SNAKE_CASE							)
											__a      :	Union[str, Any]    =  [[0] * len(__SCREAMING_SNAKE_CASE							) for x in encoded_output['input_ids']]
											__a      :	Dict    =  tokenizer.pad(__SCREAMING_SNAKE_CASE							)
											self.assertSequenceEqual(outputs['global_attention_mask']						,    __SCREAMING_SNAKE_CASE							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						pass





	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a      :	int    =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE						,    **__SCREAMING_SNAKE_CASE							)
																__a      :	Tuple    =  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE						,    **__SCREAMING_SNAKE_CASE							)
																__a      :	Optional[int]    =  'A, <mask> AllenNLP sentence.'
																__a      :	Optional[int]    =  tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE						,    add_special_tokens=__SCREAMING_SNAKE_CASE						,    return_token_type_ids=__SCREAMING_SNAKE_CASE							)
																__a      :	int    =  tokenizer_p.encode_plus(__SCREAMING_SNAKE_CASE						,    add_special_tokens=__SCREAMING_SNAKE_CASE						,    return_token_type_ids=__SCREAMING_SNAKE_CASE							)
																self.assertEqual(sum(tokens_r['token_type_ids']							)						,    sum(tokens_p['token_type_ids']							)							)
																self.assertEqual(
																    sum(tokens_r['attention_mask']							) / len(tokens_r['attention_mask']							)						,    sum(tokens_p['attention_mask']							) / len(tokens_p['attention_mask']							)						,    )

																__a      :	Union[str, Any]    =  tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']							)
																__a      :	List[str]    =  tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']							)
																self.assertSequenceEqual(tokens_p['input_ids']						,    [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)
																self.assertSequenceEqual(tokens_r['input_ids']						,    [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)

																self.assertSequenceEqual(
																    __SCREAMING_SNAKE_CASE						,    ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)
																self.assertSequenceEqual(
																    __SCREAMING_SNAKE_CASE						,    ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']							)







'''simple docstring'''



from .imports import is_rich_available


if is_rich_available():
	from rich.traceback import install

	install(show_locals=False)

else:
	raise ModuleNotFoundError('To use the rich extension, install rich with `pip install rich`')



