



'''simple docstring'''

import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	a__     (				__a,			unittest.TestCase	):
	__magic_name__		:    str	    =     LEDTokenizer
	__magic_name__		:    Optional[int]	    =     LEDTokenizerFast
	__magic_name__		:    Dict	    =     True


	def 				lowercase__  (self : Any      )     ->			int:

					"""simple docstring"""

					super().setUp()
					SCREAMING_SNAKE_CASE  :						int      					= [
					    '''l''',
					    '''o''',
					    '''w''',
					    '''e''',
					    '''r''',
					    '''s''',
					    '''t''',
					    '''i''',
					    '''d''',
					    '''n''',
					    '''\u0120''',
					    '''\u0120l''',
					    '''\u0120n''',
					    '''\u0120lo''',
					    '''\u0120low''',
					    '''er''',
					    '''\u0120lowest''',
					    '''\u0120newer''',
					    '''\u0120wider''',
					    '''<unk>''',
					]
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= dict(zip(A__,					range(len(A__      )      )      )      )
					SCREAMING_SNAKE_CASE  :						Dict      					= ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= {'''unk_token''': '''<unk>'''}

					SCREAMING_SNAKE_CASE  :						Dict      					= os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES['''vocab_file''']      )
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES['''merges_file''']      )
					with open(self.vocab_file,					'''w''',					encoding='''utf-8'''      ) as fp:
									fp.write(json.dumps(A__      ) + '''\n'''      )
					with open(self.merges_file,					'''w''',					encoding='''utf-8'''      ) as fp:
									fp.write('''\n'''.join(A__      )      )


	def 				lowercase__  (self : Union[str, Any],					**__UpperCAmelCase : Optional[int]      )     ->			int:

					"""simple docstring"""

					kwargs.update(self.special_tokens_map      )
					return self.tokenizer_class.from_pretrained(self.tmpdirname,					**A__      )


	def 				lowercase__  (self : Tuple,					**__UpperCAmelCase : Optional[int]      )     ->			List[Any]:

					"""simple docstring"""

					kwargs.update(self.special_tokens_map      )
					return self.rust_tokenizer_class.from_pretrained(self.tmpdirname,					**A__      )


	def 				lowercase__  (self : Union[str, Any],					__UpperCAmelCase : Optional[int]      )     ->			Dict:

					"""simple docstring"""

					return "lower newer", "lower newer"


	@cached_property
	def 				lowercase__  (self : str      )     ->			str:

					"""simple docstring"""

					return LEDTokenizer.from_pretrained('''allenai/led-base-16384'''      )


	@cached_property
	def 				lowercase__  (self : Optional[Any]      )     ->			Optional[int]:

					"""simple docstring"""

					return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384'''      )


	@require_torch
	def 				lowercase__  (self : Any      )     ->			str:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[str]      					= ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
					SCREAMING_SNAKE_CASE  :						List[str]      					= [0, 250, 251, 17818, 13, 39186, 1938, 4, 2]

					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						Optional[int]      					= tokenizer(A__,					max_length=len(A__      ),					padding=A__,					return_tensors='''pt'''      )
									self.assertIsInstance(A__,					A__      )

									self.assertEqual((2, 9),					batch.input_ids.shape      )
									self.assertEqual((2, 9),					batch.attention_mask.shape      )
									SCREAMING_SNAKE_CASE  :						Optional[Any]      					= batch.input_ids.tolist()[0]
									self.assertListEqual(A__,					A__      )


	@require_torch
	def 				lowercase__  (self : Optional[Any]      )     ->			Any:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= tokenizer(A__,					padding=A__,					return_tensors='''pt'''      )
									self.assertIn('''input_ids''',					A__      )
									self.assertIn('''attention_mask''',					A__      )
									self.assertNotIn('''labels''',					A__      )
									self.assertNotIn('''decoder_attention_mask''',					A__      )


	@require_torch
	def 				lowercase__  (self : List[Any]      )     ->			Optional[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= [
					    '''Summary of the text.''',
					    '''Another summary.''',
					]
					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						Tuple      					= tokenizer(text_target=A__,					max_length=32,					padding='''max_length''',					return_tensors='''pt'''      )
									self.assertEqual(32,					targets['''input_ids'''].shape[1]      )


	@require_torch
	def 				lowercase__  (self : Tuple      )     ->			List[str]:

					"""simple docstring"""

					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						str      					= tokenizer(
									    ['''I am a small frog''' * 1024, '''I am a small frog'''],					padding=A__,					truncation=A__,					return_tensors='''pt'''      )
									self.assertIsInstance(A__,					A__      )
									self.assertEqual(batch.input_ids.shape,					(2, 5122)      )


	@require_torch
	def 				lowercase__  (self : int      )     ->			Union[str, Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= ['''A long paragraph for summarization.''']
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= [
					    '''Summary of the text.''',
					]
					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						List[str]      					= tokenizer(A__,					return_tensors='''pt'''      )
									SCREAMING_SNAKE_CASE  :						str      					= tokenizer(text_target=A__,					return_tensors='''pt'''      )
									SCREAMING_SNAKE_CASE  :						int      					= inputs['''input_ids''']
									SCREAMING_SNAKE_CASE  :						List[Any]      					= targets['''input_ids''']
									self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()      )
									self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()      )
									self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()      )
									self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()      )


	@require_torch
	def 				lowercase__  (self : Any      )     ->			int:

					"""simple docstring"""

					for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
									SCREAMING_SNAKE_CASE  :						Tuple      					= ['''Summary of the text.''', '''Another summary.''']
									SCREAMING_SNAKE_CASE  :						Dict      					= [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

									SCREAMING_SNAKE_CASE  :						Dict      					= tokenizer(A__,					padding=A__      )
									SCREAMING_SNAKE_CASE  :						int      					= [[0] * len(A__      ) for x in encoded_output['''input_ids''']]
									SCREAMING_SNAKE_CASE  :						str      					= tokenizer.pad(A__      )
									self.assertSequenceEqual(outputs['''global_attention_mask'''],					A__      )


	def 				lowercase__  (self : Any      )     ->			Tuple:

					"""simple docstring"""

					pass






	def 				lowercase__  (self : Dict      )     ->			Dict:

					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
													SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= self.rust_tokenizer_class.from_pretrained(A__,					**A__      )
													SCREAMING_SNAKE_CASE  :						Optional[Any]      					= self.tokenizer_class.from_pretrained(A__,					**A__      )
													SCREAMING_SNAKE_CASE  :						Optional[Any]      					= '''A, <mask> AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE  :						Optional[int]      					= tokenizer_r.encode_plus(A__,					add_special_tokens=A__,					return_token_type_ids=A__      )
													SCREAMING_SNAKE_CASE  :						str      					= tokenizer_p.encode_plus(A__,					add_special_tokens=A__,					return_token_type_ids=A__      )
													self.assertEqual(sum(tokens_r['''token_type_ids''']      ),					sum(tokens_p['''token_type_ids''']      )      )
													self.assertEqual(
													    sum(tokens_r['''attention_mask''']      ) / len(tokens_r['''attention_mask''']      ),					sum(tokens_p['''attention_mask''']      ) / len(tokens_p['''attention_mask''']      ),					)

													SCREAMING_SNAKE_CASE  :						Dict      					= tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']      )
													SCREAMING_SNAKE_CASE  :						List[Any]      					= tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']      )
													self.assertSequenceEqual(tokens_p['''input_ids'''],					[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]      )
													self.assertSequenceEqual(tokens_r['''input_ids'''],					[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]      )

													self.assertSequenceEqual(
													    A__,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']      )
													self.assertSequenceEqual(
													    A__,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']      )









'''simple docstring'''

from ..utils import DummyObject, requires_backends


class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[Any]	    =     ["sentencepiece"]


	def __init__(self : Optional[Any],					*__UpperCAmelCase : List[Any],					**__UpperCAmelCase : List[Any]      )     ->			Optional[int]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Tuple	    =     ["sentencepiece"]


	def __init__(self : Optional[int],					*__UpperCAmelCase : int,					**__UpperCAmelCase : List[str]      )     ->			int:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : List[str],					*__UpperCAmelCase : str,					**__UpperCAmelCase : List[Any]      )     ->			str:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : Optional[int],					*__UpperCAmelCase : str,					**__UpperCAmelCase : Optional[Any]      )     ->			List[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[str]	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : Union[str, Any],					**__UpperCAmelCase : Tuple      )     ->			Any:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Union[str, Any]	    =     ["sentencepiece"]


	def __init__(self : List[Any],					*__UpperCAmelCase : List[Any],					**__UpperCAmelCase : Optional[Any]      )     ->			str:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    int	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : int,					**__UpperCAmelCase : Any      )     ->			Union[str, Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    str	    =     ["sentencepiece"]


	def __init__(self : int,					*__UpperCAmelCase : Optional[Any],					**__UpperCAmelCase : Tuple      )     ->			Optional[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    str	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : Optional[Any],					**__UpperCAmelCase : Tuple      )     ->			Dict:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[Any]	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : Optional[int],					**__UpperCAmelCase : Tuple      )     ->			int:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Tuple	    =     ["sentencepiece"]


	def __init__(self : List[Any],					*__UpperCAmelCase : Optional[int],					**__UpperCAmelCase : Optional[int]      )     ->			Union[str, Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    int	    =     ["sentencepiece"]


	def __init__(self : str,					*__UpperCAmelCase : str,					**__UpperCAmelCase : str      )     ->			Optional[int]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[str]	    =     ["sentencepiece"]


	def __init__(self : int,					*__UpperCAmelCase : List[str],					**__UpperCAmelCase : str      )     ->			Union[str, Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    str	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : Tuple,					**__UpperCAmelCase : str      )     ->			List[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Any	    =     ["sentencepiece"]


	def __init__(self : Dict,					*__UpperCAmelCase : Optional[int],					**__UpperCAmelCase : List[str]      )     ->			Optional[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : str,					*__UpperCAmelCase : Optional[int],					**__UpperCAmelCase : Tuple      )     ->			Dict:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[Any]	    =     ["sentencepiece"]


	def __init__(self : Union[str, Any],					*__UpperCAmelCase : int,					**__UpperCAmelCase : str      )     ->			Optional[int]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : Any,					*__UpperCAmelCase : str,					**__UpperCAmelCase : List[Any]      )     ->			Optional[int]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : List[Any],					*__UpperCAmelCase : Union[str, Any],					**__UpperCAmelCase : List[Any]      )     ->			List[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : Dict,					*__UpperCAmelCase : Optional[Any],					**__UpperCAmelCase : str      )     ->			List[Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[Any]	    =     ["sentencepiece"]


	def __init__(self : List[Any],					*__UpperCAmelCase : Any,					**__UpperCAmelCase : str      )     ->			int:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    str	    =     ["sentencepiece"]


	def __init__(self : Optional[int],					*__UpperCAmelCase : str,					**__UpperCAmelCase : Tuple      )     ->			Any:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[int]	    =     ["sentencepiece"]


	def __init__(self : Union[str, Any],					*__UpperCAmelCase : Dict,					**__UpperCAmelCase : Optional[int]      )     ->			Tuple:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Tuple	    =     ["sentencepiece"]


	def __init__(self : List[str],					*__UpperCAmelCase : Union[str, Any],					**__UpperCAmelCase : Dict      )     ->			List[str]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Any	    =     ["sentencepiece"]


	def __init__(self : int,					*__UpperCAmelCase : Union[str, Any],					**__UpperCAmelCase : Optional[Any]      )     ->			Union[str, Any]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[Any]	    =     ["sentencepiece"]


	def __init__(self : Union[str, Any],					*__UpperCAmelCase : Tuple,					**__UpperCAmelCase : List[str]      )     ->			Optional[int]:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Any	    =     ["sentencepiece"]


	def __init__(self : Optional[int],					*__UpperCAmelCase : str,					**__UpperCAmelCase : str      )     ->			str:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    int	    =     ["sentencepiece"]


	def __init__(self : Dict,					*__UpperCAmelCase : Any,					**__UpperCAmelCase : List[str]      )     ->			Dict:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    Optional[Any]	    =     ["sentencepiece"]


	def __init__(self : List[Any],					*__UpperCAmelCase : Optional[Any],					**__UpperCAmelCase : int      )     ->			Any:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    int	    =     ["sentencepiece"]


	def __init__(self : Tuple,					*__UpperCAmelCase : Optional[Any],					**__UpperCAmelCase : int      )     ->			int:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )



class 	a__     (				metaclass=_lowercase	):
	__magic_name__		:    List[Any]	    =     ["sentencepiece"]


	def __init__(self : Optional[int],					*__UpperCAmelCase : str,					**__UpperCAmelCase : List[str]      )     ->			str:

					"""simple docstring"""

					requires_backends(self,					['''sentencepiece''']      )




