




'''simple docstring'''


# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
_snake_case : List[Any]  				=						abspath(join(dirname(__file__), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)
def 						snake_case_       (UpperCamelCase			:    Union[str, Any] ):





							'''simple docstring'''


							config.addinivalue_line(
							    '''markers'''					,		'''is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested''' )
							config.addinivalue_line(
							    '''markers'''					,		'''is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested''' )
							config.addinivalue_line('''markers'''					,		'''is_pipeline_test: mark test to run only when pipelines are tested''' )
							config.addinivalue_line('''markers'''					,		'''is_staging_test: mark test to run only in the staging environment''' )
							config.addinivalue_line('''markers'''					,		'''accelerate_tests: mark test that require accelerate''' )
							config.addinivalue_line('''markers'''					,		'''tool_tests: mark the tool tests that are run on their specific schedule''' )
def 						snake_case_       (UpperCamelCase			:    Optional[Any] ):





							'''simple docstring'''


							from transformers.testing_utils import pytest_addoption_shared

							pytest_addoption_shared(UpperCamelCase )
def 						snake_case_       (UpperCamelCase			:    Optional[int] ):





							'''simple docstring'''


							from transformers.testing_utils import pytest_terminal_summary_main

							_a      	=							terminalreporter.config.getoption('''--make-reports''' )
							if make_reports:
														pytest_terminal_summary_main(UpperCamelCase					,		id=UpperCamelCase )
def 						snake_case_       (UpperCamelCase			:    Any					,		UpperCamelCase			:    str ):





							'''simple docstring'''


							if exitstatus == 5:
														_a      	=							0


# Doctest custom flag to ignore output.
_snake_case : Union[str, Any]  				=						doctest.register_optionflag('IGNORE_RESULT')

_snake_case : Union[str, Any]  				=						doctest.OutputChecker

class      A       (						_a       ):







							def   __lowerCAmelCase     (    self			:    Any				,     lowerCAmelCase_			:    Dict				,     lowerCAmelCase_			:    Tuple				,     lowerCAmelCase_			:    Optional[Any]   )     ->				List[str]:


														"""simple docstring"""


														if IGNORE_RESULT & optionflags:
																					return True
														return OutputChecker.check_output(self				,     lowerCAmelCase_				,     lowerCAmelCase_				,     lowerCAmelCase_   )



_snake_case : Tuple  				=						CustomOutputChecker
_snake_case : Optional[Any]  				=						HfDoctestModule

_snake_case : str  				=						HfDocTestParser







'''simple docstring'''


import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      A       (						_a	,unittest.TestCase       ):
							lowercase_							     =							LEDTokenizer
							lowercase_							     =							LEDTokenizerFast
							lowercase_							     =							True







							def   __lowerCAmelCase     (    self			:    int   )     ->				List[Any]:


														"""simple docstring"""


														super().setUp()
														_a      	=							[
														    '''l''',
														    '''o''',
														    '''w''',
														    '''e''',
														    '''r''',
														    '''s''',
														    '''t''',
														    '''i''',
														    '''d''',
														    '''n''',
														    '''\u0120''',
														    '''\u0120l''',
														    '''\u0120n''',
														    '''\u0120lo''',
														    '''\u0120low''',
														    '''er''',
														    '''\u0120lowest''',
														    '''\u0120newer''',
														    '''\u0120wider''',
														    '''<unk>''',
														]
														_a      	=							dict(zip(lowerCAmelCase_				,     range(len(lowerCAmelCase_   )   )   )   )
														_a      	=							['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
														_a      	=							{'''unk_token''': '''<unk>'''}

														_a      	=							os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['''vocab_file''']   )
														_a      	=							os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['''merges_file''']   )
														with open(self.vocab_file				,     '''w'''				,     encoding='''utf-8'''   ) as fp:
																					fp.write(json.dumps(lowerCAmelCase_   ) + '''\n'''   )
														with open(self.merges_file				,     '''w'''				,     encoding='''utf-8'''   ) as fp:
																					fp.write('''\n'''.join(lowerCAmelCase_   )   )







							def   __lowerCAmelCase     (    self			:    Union[str, Any]				,     **lowerCAmelCase_			:    int   )     ->				Optional[int]:


														"""simple docstring"""


														kwargs.update(self.special_tokens_map   )
														return self.tokenizer_class.from_pretrained(self.tmpdirname				,     **lowerCAmelCase_   )







							def   __lowerCAmelCase     (    self			:    Optional[Any]				,     **lowerCAmelCase_			:    Any   )     ->				int:


														"""simple docstring"""


														kwargs.update(self.special_tokens_map   )
														return self.rust_tokenizer_class.from_pretrained(self.tmpdirname				,     **lowerCAmelCase_   )







							def   __lowerCAmelCase     (    self			:    Optional[Any]				,     lowerCAmelCase_			:    Dict   )     ->				List[str]:


														"""simple docstring"""


														return "lower newer", "lower newer"







							@cached_property
							def   __lowerCAmelCase     (    self			:    Dict   )     ->				int:


														"""simple docstring"""


														return LEDTokenizer.from_pretrained('''allenai/led-base-16384'''   )







							@cached_property
							def   __lowerCAmelCase     (    self			:    List[str]   )     ->				Union[str, Any]:


														"""simple docstring"""


														return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384'''   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    int   )     ->				Tuple:


														"""simple docstring"""


														_a      	=							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
														_a      	=							[0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							tokenizer(lowerCAmelCase_				,     max_length=len(lowerCAmelCase_   )				,     padding=lowerCAmelCase_				,     return_tensors='''pt'''   )
																					self.assertIsInstance(lowerCAmelCase_				,     lowerCAmelCase_   )

																					self.assertEqual((2, 9)				,     batch.input_ids.shape   )
																					self.assertEqual((2, 9)				,     batch.attention_mask.shape   )
																					_a      	=							batch.input_ids.tolist()[0]
																					self.assertListEqual(lowerCAmelCase_				,     lowerCAmelCase_   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    Tuple   )     ->				List[Any]:


														"""simple docstring"""


														_a      	=							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							tokenizer(lowerCAmelCase_				,     padding=lowerCAmelCase_				,     return_tensors='''pt'''   )
																					self.assertIn('''input_ids'''				,     lowerCAmelCase_   )
																					self.assertIn('''attention_mask'''				,     lowerCAmelCase_   )
																					self.assertNotIn('''labels'''				,     lowerCAmelCase_   )
																					self.assertNotIn('''decoder_attention_mask'''				,     lowerCAmelCase_   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    List[str]   )     ->				str:


														"""simple docstring"""


														_a      	=							[
														    '''Summary of the text.''',
														    '''Another summary.''',
														]
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							tokenizer(text_target=lowerCAmelCase_				,     max_length=32				,     padding='''max_length'''				,     return_tensors='''pt'''   )
																					self.assertEqual(32				,     targets['''input_ids'''].shape[1]   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    Any   )     ->				Union[str, Any]:


														"""simple docstring"""


														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							tokenizer(
																					    ['''I am a small frog''' * 10_24, '''I am a small frog''']				,     padding=lowerCAmelCase_				,     truncation=lowerCAmelCase_				,     return_tensors='''pt'''   )
																					self.assertIsInstance(lowerCAmelCase_				,     lowerCAmelCase_   )
																					self.assertEqual(batch.input_ids.shape				,     (2, 51_22)   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    Optional[Any]   )     ->				Union[str, Any]:


														"""simple docstring"""


														_a      	=							['''A long paragraph for summarization.''']
														_a      	=							[
														    '''Summary of the text.''',
														]
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							tokenizer(lowerCAmelCase_				,     return_tensors='''pt'''   )
																					_a      	=							tokenizer(text_target=lowerCAmelCase_				,     return_tensors='''pt'''   )
																					_a      	=							inputs['''input_ids''']
																					_a      	=							targets['''input_ids''']
																					self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()   )
																					self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()   )
																					self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()   )
																					self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()   )







							@require_torch
							def   __lowerCAmelCase     (    self			:    Any   )     ->				Union[str, Any]:


														"""simple docstring"""


														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					_a      	=							['''Summary of the text.''', '''Another summary.''']
																					_a      	=							[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

																					_a      	=							tokenizer(lowerCAmelCase_				,     padding=lowerCAmelCase_   )
																					_a      	=							[[0] * len(lowerCAmelCase_   ) for x in encoded_output['''input_ids''']]
																					_a      	=							tokenizer.pad(lowerCAmelCase_   )
																					self.assertSequenceEqual(outputs['''global_attention_mask''']				,     lowerCAmelCase_   )







							def   __lowerCAmelCase     (    self			:    Any   )     ->				Dict:


														"""simple docstring"""


														pass







							def   __lowerCAmelCase     (    self			:    Any   )     ->				Optional[Any]:


														"""simple docstring"""


														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
																												_a      	=							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,     **lowerCAmelCase_   )
																												_a      	=							self.tokenizer_class.from_pretrained(lowerCAmelCase_				,     **lowerCAmelCase_   )
																												_a      	=							'''A, <mask> AllenNLP sentence.'''
																												_a      	=							tokenizer_r.encode_plus(lowerCAmelCase_				,     add_special_tokens=lowerCAmelCase_				,     return_token_type_ids=lowerCAmelCase_   )
																												_a      	=							tokenizer_p.encode_plus(lowerCAmelCase_				,     add_special_tokens=lowerCAmelCase_				,     return_token_type_ids=lowerCAmelCase_   )
																												self.assertEqual(sum(tokens_r['''token_type_ids''']   )				,     sum(tokens_p['''token_type_ids''']   )   )
																												self.assertEqual(
																												    sum(tokens_r['''attention_mask''']   ) / len(tokens_r['''attention_mask''']   )				,     sum(tokens_p['''attention_mask''']   ) / len(tokens_p['''attention_mask''']   )				,     )

																												_a      	=							tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']   )
																												_a      	=							tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']   )
																												self.assertSequenceEqual(tokens_p['''input_ids''']				,     [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )
																												self.assertSequenceEqual(tokens_r['''input_ids''']				,     [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )

																												self.assertSequenceEqual(
																												    lowerCAmelCase_				,     ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )
																												self.assertSequenceEqual(
																												    lowerCAmelCase_				,     ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )

