


from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel







def 				UpperCAmelCase					(  UpperCAmelCase					,UpperCAmelCase					,UpperCAmelCase=None					,UpperCAmelCase=None   )->		List[Any]:







					'''simple docstring'''



					if attention_mask is None:
										SCREAMING_SNAKE_CASE_						   =							tf.cast(tf.math.not_equal(_lowerCamelCase					,config.pad_token_id   )					,tf.inta   )
					return {"input_ids": input_ids, "attention_mask": attention_mask}

@require_tf
class     snake_case      :
			'''simple docstring'''

			UpperCAmelCase							:							List[Any]							=			OPTConfig
			UpperCAmelCase							:							List[str]							=			{}
			UpperCAmelCase							:							Tuple							=			'''gelu'''

			def __init__(						self  :  Tuple				,				lowerCAmelCase_  :  List[Any]				,				lowerCAmelCase_  :  Optional[Any]=13				,				lowerCAmelCase_  :  Optional[Any]=7				,				lowerCAmelCase_  :  List[Any]=True				,				lowerCAmelCase_  :  List[str]=False				,				lowerCAmelCase_  :  str=99				,				lowerCAmelCase_  :  List[str]=16				,				lowerCAmelCase_  :  List[Any]=2				,				lowerCAmelCase_  :  Optional[Any]=4				,				lowerCAmelCase_  :  Dict=4				,				lowerCAmelCase_  :  Optional[int]="gelu"				,				lowerCAmelCase_  :  List[str]=0.1				,				lowerCAmelCase_  :  str=0.1				,				lowerCAmelCase_  :  Dict=20				,				lowerCAmelCase_  :  str=2				,				lowerCAmelCase_  :  Dict=1				,				lowerCAmelCase_  :  Optional[int]=0				,				lowerCAmelCase_  :  str=16				,				lowerCAmelCase_  :  int=16				,				)       ->       Tuple:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							parent
								SCREAMING_SNAKE_CASE_						   =							batch_size
								SCREAMING_SNAKE_CASE_						   =							seq_length
								SCREAMING_SNAKE_CASE_						   =							is_training
								SCREAMING_SNAKE_CASE_						   =							use_labels
								SCREAMING_SNAKE_CASE_						   =							vocab_size
								SCREAMING_SNAKE_CASE_						   =							hidden_size
								SCREAMING_SNAKE_CASE_						   =							num_hidden_layers
								SCREAMING_SNAKE_CASE_						   =							num_attention_heads
								SCREAMING_SNAKE_CASE_						   =							intermediate_size
								SCREAMING_SNAKE_CASE_						   =							hidden_act
								SCREAMING_SNAKE_CASE_						   =							hidden_dropout_prob
								SCREAMING_SNAKE_CASE_						   =							attention_probs_dropout_prob
								SCREAMING_SNAKE_CASE_						   =							max_position_embeddings
								SCREAMING_SNAKE_CASE_						   =							eos_token_id
								SCREAMING_SNAKE_CASE_						   =							pad_token_id
								SCREAMING_SNAKE_CASE_						   =							bos_token_id
								SCREAMING_SNAKE_CASE_						   =							embed_dim
								SCREAMING_SNAKE_CASE_						   =							word_embed_proj_dim
								SCREAMING_SNAKE_CASE_						   =							False

			def _lowercase			(						self  :  List[Any]       )       ->       str:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							ids_tensor([self.batch_size, self.seq_length - 1]				,				self.vocab_size       )
								SCREAMING_SNAKE_CASE_						   =							tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )				,				1       )
								SCREAMING_SNAKE_CASE_						   =							tf.concat([input_ids, eos_tensor]				,				axis=1       )

								SCREAMING_SNAKE_CASE_						   =							self.config_cls(
								    vocab_size=self.vocab_size				,				hidden_size=self.hidden_size				,				num_hidden_layers=self.num_hidden_layers				,				num_attention_heads=self.num_attention_heads				,				ffn_dim=self.intermediate_size				,				dropout=self.hidden_dropout_prob				,				attention_dropout=self.attention_probs_dropout_prob				,				max_position_embeddings=self.max_position_embeddings				,				eos_token_id=self.eos_token_id				,				bos_token_id=self.bos_token_id				,				pad_token_id=self.pad_token_id				,				embed_dim=self.embed_dim				,				word_embed_proj_dim=self.word_embed_proj_dim				,				is_encoder_decoder=__SCREAMING_SNAKE_CASE				,				**self.config_updates				,				)
								SCREAMING_SNAKE_CASE_						   =							prepare_opt_inputs_dict(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE       )
								return config, inputs_dict




			def _lowercase			(						self  :  Any				,				lowerCAmelCase_  :  List[Any]				,				lowerCAmelCase_  :  List[str]       )       ->       List[str]:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							TFOPTModel(config=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							inputs_dict['''input_ids''']

								SCREAMING_SNAKE_CASE_						   =							input_ids[:1, :]
								SCREAMING_SNAKE_CASE_						   =							inputs_dict['''attention_mask'''][:1, :]
								SCREAMING_SNAKE_CASE_						   =							1

								# first forward pass
								SCREAMING_SNAKE_CASE_						   =							model(__SCREAMING_SNAKE_CASE				,				attention_mask=__SCREAMING_SNAKE_CASE				,				use_cache=__SCREAMING_SNAKE_CASE       )

								SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								SCREAMING_SNAKE_CASE_						   =							ids_tensor((self.batch_size, 3)				,				config.vocab_size       )
								SCREAMING_SNAKE_CASE_						   =							tf.cast(ids_tensor((self.batch_size, 3)				,				2       )				,				tf.inta       )

								# append to next input_ids and
								SCREAMING_SNAKE_CASE_						   =							tf.concat([input_ids, next_tokens]				,				axis=-1       )
								SCREAMING_SNAKE_CASE_						   =							tf.concat([attention_mask, next_attn_mask]				,				axis=-1       )

								SCREAMING_SNAKE_CASE_						   =							model(__SCREAMING_SNAKE_CASE				,				attention_mask=__SCREAMING_SNAKE_CASE       )[0]
								SCREAMING_SNAKE_CASE_						   =							model(__SCREAMING_SNAKE_CASE				,				attention_mask=__SCREAMING_SNAKE_CASE				,				past_key_values=__SCREAMING_SNAKE_CASE       )[0]

								self.parent.assertEqual(next_tokens.shape[1]				,				output_from_past.shape[1]       )

								# select random slice
								SCREAMING_SNAKE_CASE_						   =							int(ids_tensor((1,)				,				output_from_past.shape[-1]       )       )
								SCREAMING_SNAKE_CASE_						   =							output_from_no_past[:, -3:, random_slice_idx]
								SCREAMING_SNAKE_CASE_						   =							output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				rtol=1e-3       )

@require_tf
class     snake_case      (							__lowerCAmelCase						,			__lowerCAmelCase						,			unittest.TestCase						):
			'''simple docstring'''

			UpperCAmelCase							:							Optional[int]							=			(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
			UpperCAmelCase							:							Tuple							=			(TFOPTForCausalLM,) if is_tf_available() else ()
			UpperCAmelCase							:							int							=			(
			    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
			)
			UpperCAmelCase							:							Tuple							=			False
			UpperCAmelCase							:							int							=			False
			UpperCAmelCase							:							Any							=			False
			UpperCAmelCase							:							Optional[Any]							=			10

			def _lowercase			(						self  :  Any       )       ->       Any:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							TFOPTModelTester(self       )
								SCREAMING_SNAKE_CASE_						   =							ConfigTester(self				,				config_class=__SCREAMING_SNAKE_CASE       )

			def _lowercase			(						self  :  int       )       ->       List[str]:




								"""simple docstring"""






								self.config_tester.run_common_tests()

			def _lowercase			(						self  :  Union[str, Any]       )       ->       Any:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*__SCREAMING_SNAKE_CASE       )




			def _lowercase			(						self  :  List[str]       )       ->       Tuple:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							self.model_tester.prepare_config_and_inputs_for_common()

								def _get_word_embedding_weight(lowerCAmelCase_  :  Union[str, Any]				,				lowerCAmelCase_  :  int       ):
													if hasattr(__SCREAMING_SNAKE_CASE				,				'''weight'''       ):
																		return embedding_layer.weight
													else:
																		# Here we build the word embeddings weights if not exists.
																		# And then we retry to get the attribute once built.
																		model.build()
																		if hasattr(__SCREAMING_SNAKE_CASE				,				'''weight'''       ):
																							return embedding_layer.weight
																		else:
																							return None

								for model_class in self.all_model_classes:
													for size in [config.vocab_size - 10, config.vocab_size + 10]:
																		# build the embeddings
																		SCREAMING_SNAKE_CASE_						   =							model_class(config=__SCREAMING_SNAKE_CASE       )
																		SCREAMING_SNAKE_CASE_						   =							_get_word_embedding_weight(__SCREAMING_SNAKE_CASE				,				model.get_input_embeddings()       )
																		SCREAMING_SNAKE_CASE_						   =							_get_word_embedding_weight(__SCREAMING_SNAKE_CASE				,				model.get_output_embeddings()       )

																		# reshape the embeddings
																		model.resize_token_embeddings(__SCREAMING_SNAKE_CASE       )
																		SCREAMING_SNAKE_CASE_						   =							_get_word_embedding_weight(__SCREAMING_SNAKE_CASE				,				model.get_input_embeddings()       )
																		SCREAMING_SNAKE_CASE_						   =							_get_word_embedding_weight(__SCREAMING_SNAKE_CASE				,				model.get_output_embeddings()       )

																		# check that the resized embeddings size matches the desired size.
																		SCREAMING_SNAKE_CASE_						   =							size if size is not None else config.vocab_size

																		self.assertEqual(new_input_embeddings.shape[0]				,				__SCREAMING_SNAKE_CASE       )

																		# check that weights remain the same after resizing
																		SCREAMING_SNAKE_CASE_						   =							True
																		for pa, pa in zip(old_input_embeddings.value()				,				new_input_embeddings.value()       ):
																							if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																												SCREAMING_SNAKE_CASE_						   =							False
																		self.assertTrue(__SCREAMING_SNAKE_CASE       )

																		if old_output_embeddings is not None and new_output_embeddings is not None:
																							self.assertEqual(new_output_embeddings.shape[0]				,				__SCREAMING_SNAKE_CASE       )

																							SCREAMING_SNAKE_CASE_						   =							True
																							for pa, pa in zip(old_output_embeddings.value()				,				new_output_embeddings.value()       ):
																												if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																																	SCREAMING_SNAKE_CASE_						   =							False
																							self.assertTrue(__SCREAMING_SNAKE_CASE       )







def 				UpperCAmelCase					(  UpperCAmelCase   )->		List[str]:







					'''simple docstring'''



					return tf.constant(_lowerCamelCase					,dtype=tf.intaa   )

@require_tf
class     snake_case      (							unittest.TestCase						):
			'''simple docstring'''

			UpperCAmelCase							:							Dict							=			99

			def _lowercase			(						self  :  List[str]       )       ->       str:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							tf.ones((4, 1)				,				dtype=tf.intaa       ) * 2
								SCREAMING_SNAKE_CASE_						   =							tf.concat([ids_tensor((4, 6)				,				self.vocab_size - 3       ) + 3, eos_column_vector]				,				axis=1       )
								SCREAMING_SNAKE_CASE_						   =							input_ids.shape[0]
								SCREAMING_SNAKE_CASE_						   =							OPTConfig(
								    vocab_size=self.vocab_size				,				hidden_size=24				,				num_hidden_layers=2				,				num_attention_heads=2				,				ffn_dim=32				,				max_position_embeddings=48				,				eos_token_id=2				,				pad_token_id=1				,				bos_token_id=0				,				)
								return config, input_ids, batch_size

@require_sentencepiece
@require_tf
class     snake_case      (							unittest.TestCase						):
			'''simple docstring'''

			@slow
			def _lowercase			(						self  :  Tuple       )       ->       Dict:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							TFOPTModel.from_pretrained('''facebook/opt-350m'''       )
								SCREAMING_SNAKE_CASE_						   =							_long_tensor([[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]]       )
								SCREAMING_SNAKE_CASE_						   =							tf.not_equal(__SCREAMING_SNAKE_CASE				,				model.config.pad_token_id       )
								with tf.GradientTape():
													SCREAMING_SNAKE_CASE_						   =							model(input_ids=__SCREAMING_SNAKE_CASE				,				attention_mask=__SCREAMING_SNAKE_CASE       ).last_hidden_state
								SCREAMING_SNAKE_CASE_						   =							(1, 11, 512)
								self.assertEqual(output.shape				,				__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							tf.constant(
								    [[-0.2_873, -1.9_218, -0.3_033], [-1.2_710, -0.1_338, -0.1_902], [0.4_095, 0.1_214, -1.3_121]]       )
								self.assertTrue(np.allclose(output[:, :3, :3]				,				__SCREAMING_SNAKE_CASE				,				atol=4e-3       )       )

								SCREAMING_SNAKE_CASE_						   =							tf.function(__SCREAMING_SNAKE_CASE				,				jit_compile=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							xla_generate(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE       )[0]
								self.assertTrue(np.allclose(output[:, :3, :3]				,				__SCREAMING_SNAKE_CASE				,				atol=4e-2       )       )

@require_tf
@slow
class     snake_case      (							unittest.TestCase						):
			'''simple docstring'''

			def _lowercase			(						self  :  int       )       ->       Optional[Any]:




								"""simple docstring"""






								super().setUp()
								SCREAMING_SNAKE_CASE_						   =							'''facebook/opt-350m'''




			def _lowercase			(						self  :  Optional[Any]       )       ->       Union[str, Any]:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							TFOPTForCausalLM.from_pretrained(self.path_model       )
								SCREAMING_SNAKE_CASE_						   =							GPTaTokenizer.from_pretrained(self.path_model       )

								SCREAMING_SNAKE_CASE_						   =							[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of''',
								    '''Paris is the capital of France and''',
								    '''Computers and mobile phones have taken''',
								]
								# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
								SCREAMING_SNAKE_CASE_						   =							tokenizer(__SCREAMING_SNAKE_CASE				,				return_tensors='''tf'''				,				padding=__SCREAMING_SNAKE_CASE				,				add_special_tokens=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							tf.math.reduce_mean(model(inputs.input_ids				,				attention_mask=inputs.attention_mask       )[0]				,				axis=-1       )
								SCREAMING_SNAKE_CASE_						   =							tf.constant(
								    [
								        [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
								        [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
								        [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
								        [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
								    ]       )
								self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				atol=1e-4       )       )

								SCREAMING_SNAKE_CASE_						   =							tf.function(__SCREAMING_SNAKE_CASE				,				jit_compile=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							tf.math.reduce_mean(xla_generate(inputs.input_ids				,				attention_mask=inputs.attention_mask       )[0]				,				axis=-1       )
								self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				atol=1e-4       )       )

@require_tf
@slow
class     snake_case      (							unittest.TestCase						):
			'''simple docstring'''

			@property
			def _lowercase			(						self  :  Union[str, Any]       )       ->       Dict:




								"""simple docstring"""






								return [
								    "Today is a beautiful day and I want",
								    "In the city of",
								    "Paris is the capital of France and",
								    "Computers and mobile phones have taken",
								]

			def _lowercase			(						self  :  str       )       ->       List[str]:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							'''facebook/opt-125m'''

								SCREAMING_SNAKE_CASE_						   =							[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of New York, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								SCREAMING_SNAKE_CASE_						   =							[]
								SCREAMING_SNAKE_CASE_						   =							GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE       )

								for prompt in self.prompts:
													SCREAMING_SNAKE_CASE_						   =							tokenizer(__SCREAMING_SNAKE_CASE				,				return_tensors='''tf'''       ).input_ids

													SCREAMING_SNAKE_CASE_						   =							model.generate(__SCREAMING_SNAKE_CASE				,				max_length=10       )

													SCREAMING_SNAKE_CASE_						   =							tokenizer.batch_decode(__SCREAMING_SNAKE_CASE				,				skip_special_tokens=__SCREAMING_SNAKE_CASE       )
													predicted_outputs += generated_string

								self.assertListEqual(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE       )

			def _lowercase			(						self  :  Tuple       )       ->       List[Any]:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							'''facebook/opt-350m'''

								SCREAMING_SNAKE_CASE_						   =							GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE       )

								SCREAMING_SNAKE_CASE_						   =							'''left'''

								# use different length sentences to test batching
								SCREAMING_SNAKE_CASE_						   =							[
								    '''Hello, my dog is a little''',
								    '''Today, I''',
								]

								SCREAMING_SNAKE_CASE_						   =							tokenizer(__SCREAMING_SNAKE_CASE				,				return_tensors='''tf'''				,				padding=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							inputs['''input_ids''']

								SCREAMING_SNAKE_CASE_						   =							model.generate(input_ids=__SCREAMING_SNAKE_CASE				,				attention_mask=inputs['''attention_mask''']       )

								SCREAMING_SNAKE_CASE_						   =							tokenizer(sentences[0]				,				return_tensors='''tf'''       ).input_ids
								SCREAMING_SNAKE_CASE_						   =							model.generate(input_ids=__SCREAMING_SNAKE_CASE       )

								SCREAMING_SNAKE_CASE_						   =							inputs_non_padded.shape[-1] - tf.math.reduce_sum(
								    tf.cast(inputs['''attention_mask'''][-1]				,				tf.intaa       )       )
								SCREAMING_SNAKE_CASE_						   =							tokenizer(sentences[1]				,				return_tensors='''tf'''       ).input_ids
								SCREAMING_SNAKE_CASE_						   =							model.generate(input_ids=__SCREAMING_SNAKE_CASE				,				max_length=model.config.max_length - num_paddings       )

								SCREAMING_SNAKE_CASE_						   =							tokenizer.batch_decode(__SCREAMING_SNAKE_CASE				,				skip_special_tokens=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							tokenizer.decode(output_non_padded[0]				,				skip_special_tokens=__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							tokenizer.decode(output_padded[0]				,				skip_special_tokens=__SCREAMING_SNAKE_CASE       )

								SCREAMING_SNAKE_CASE_						   =							[
								    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
								    '''Today, I was in the middle of a conversation with a friend about the''',
								]
								self.assertListEqual(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE       )
								self.assertListEqual(__SCREAMING_SNAKE_CASE				,				[non_padded_sentence, padded_sentence]       )




			def _lowercase			(						self  :  List[str]       )       ->       Dict:




								"""simple docstring"""






								SCREAMING_SNAKE_CASE_						   =							'''facebook/opt-350m'''

								SCREAMING_SNAKE_CASE_						   =							[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of San Francisco, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								SCREAMING_SNAKE_CASE_						   =							[]
								SCREAMING_SNAKE_CASE_						   =							GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE_						   =							TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE       )

								for prompt in self.prompts:
													SCREAMING_SNAKE_CASE_						   =							tokenizer(__SCREAMING_SNAKE_CASE				,				return_tensors='''tf'''       ).input_ids

													SCREAMING_SNAKE_CASE_						   =							model.generate(__SCREAMING_SNAKE_CASE				,				max_length=10       )

													SCREAMING_SNAKE_CASE_						   =							tokenizer.batch_decode(__SCREAMING_SNAKE_CASE				,				skip_special_tokens=__SCREAMING_SNAKE_CASE       )
													predicted_outputs += generated_string

								self.assertListEqual(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE       )








'''simple docstring'''



import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


UpperCAmelCase_							:	Any	     =		logging.get_logger(__name__)


UpperCAmelCase_							:	str	     =		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase_							:	str	     =		{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

UpperCAmelCase_							:	Union[str, Any]	     =		{
    '''allenai/led-base-16384''': 1_6_3_8_4,
}





class      lowerCAmelCase (			__lowerCAmelCase):
       __lowercase	:   Dict										=      VOCAB_FILES_NAMES
       __lowercase	:   Dict										=      PRETRAINED_VOCAB_FILES_MAP
       __lowercase	:   Tuple										=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowercase	:   Union[str, Any]										=      LEDTokenizer
       __lowercase	:   int										=      ['''input_ids''', '''attention_mask''']


       def __init__( self     ,	__SCREAMING_SNAKE_CASE=None     ,	__SCREAMING_SNAKE_CASE=None     ,	__SCREAMING_SNAKE_CASE=None     ,	__SCREAMING_SNAKE_CASE="replace"     ,	__SCREAMING_SNAKE_CASE="<s>"     ,	__SCREAMING_SNAKE_CASE="</s>"     ,	__SCREAMING_SNAKE_CASE="</s>"     ,	__SCREAMING_SNAKE_CASE="<s>"     ,	__SCREAMING_SNAKE_CASE="<unk>"     ,	__SCREAMING_SNAKE_CASE="<pad>"     ,	__SCREAMING_SNAKE_CASE="<mask>"     ,	__SCREAMING_SNAKE_CASE=False     ,	__SCREAMING_SNAKE_CASE=True     ,	**__SCREAMING_SNAKE_CASE     ,	)   ->	List[Any]:



            '''simple docstring'''




            super().__init__(
                __SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE     ,	tokenizer_file=__SCREAMING_SNAKE_CASE     ,	errors=__SCREAMING_SNAKE_CASE     ,	bos_token=__SCREAMING_SNAKE_CASE     ,	eos_token=__SCREAMING_SNAKE_CASE     ,	sep_token=__SCREAMING_SNAKE_CASE     ,	cls_token=__SCREAMING_SNAKE_CASE     ,	unk_token=__SCREAMING_SNAKE_CASE     ,	pad_token=__SCREAMING_SNAKE_CASE     ,	mask_token=__SCREAMING_SNAKE_CASE     ,	add_prefix_space=__SCREAMING_SNAKE_CASE     ,	trim_offsets=__SCREAMING_SNAKE_CASE     ,	**__SCREAMING_SNAKE_CASE     ,	)

            __snake_case            =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
            if pre_tok_state.get('''add_prefix_space'''     ,	__SCREAMING_SNAKE_CASE	) != add_prefix_space:
                 __snake_case            =	getattr(__SCREAMING_SNAKE_CASE     ,	pre_tok_state.pop('''type'''	)	)
                 __snake_case            =	add_prefix_space
                 __snake_case            =	pre_tok_class(**__SCREAMING_SNAKE_CASE	)

            __snake_case            =	add_prefix_space

            # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
            __snake_case            =	'''post_processor'''
            __snake_case            =	getattr(self.backend_tokenizer     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)
            if tokenizer_component_instance:
                 __snake_case            =	json.loads(tokenizer_component_instance.__getstate__()	)

                 # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                 if "sep" in state:
                      __snake_case            =	tuple(state['''sep''']	)
                 if "cls" in state:
                      __snake_case            =	tuple(state['''cls''']	)

                 __snake_case            =	False

                 if state.get('''add_prefix_space'''     ,	__SCREAMING_SNAKE_CASE	) != add_prefix_space:
                      __snake_case            =	add_prefix_space
                      __snake_case            =	True

                 if state.get('''trim_offsets'''     ,	__SCREAMING_SNAKE_CASE	) != trim_offsets:
                      __snake_case            =	trim_offsets
                      __snake_case            =	True

                 if changes_to_apply:
                      __snake_case            =	getattr(__SCREAMING_SNAKE_CASE     ,	state.pop('''type'''	)	)
                      __snake_case            =	component_class(**__SCREAMING_SNAKE_CASE	)
                      setattr(self.backend_tokenizer     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)


       @property
       # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
       def    lowerCAmelCase   ( self	)   ->	str:



            '''simple docstring'''




            if self._mask_token is None:
                 if self.verbose:
                      logger.error('''Using mask_token, but it is not set yet.'''	)
                 return None
            return str(self._mask_token	)


       @mask_token.setter
       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE	)   ->	Dict:



            '''simple docstring'''




            __snake_case            =	AddedToken(__SCREAMING_SNAKE_CASE     ,	lstrip=__SCREAMING_SNAKE_CASE     ,	rstrip=__SCREAMING_SNAKE_CASE	) if isinstance(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	) else value
            __snake_case            =	value


       def    lowerCAmelCase   ( self     ,	*__SCREAMING_SNAKE_CASE     ,	**__SCREAMING_SNAKE_CASE	)   ->	BatchEncoding:



            '''simple docstring'''




            __snake_case            =	kwargs.get('''is_split_into_words'''     ,	__SCREAMING_SNAKE_CASE	)

            if is_split_into_words and not self.add_prefix_space:
                 raise ValueError(
                     F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                     '''to use it with pretokenized inputs.'''	)

            return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE     ,	**__SCREAMING_SNAKE_CASE	)


       def    lowerCAmelCase   ( self     ,	*__SCREAMING_SNAKE_CASE     ,	**__SCREAMING_SNAKE_CASE	)   ->	BatchEncoding:



            '''simple docstring'''




            __snake_case            =	kwargs.get('''is_split_into_words'''     ,	__SCREAMING_SNAKE_CASE	)

            if is_split_into_words and not self.add_prefix_space:
                 raise ValueError(
                     F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                     '''to use it with pretokenized inputs.'''	)

            return super()._encode_plus(*__SCREAMING_SNAKE_CASE     ,	**__SCREAMING_SNAKE_CASE	)


       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE = None	)   ->	Tuple[str]:



            '''simple docstring'''




            __snake_case            =	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,	name=__SCREAMING_SNAKE_CASE	)
            return tuple(__SCREAMING_SNAKE_CASE	)


       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE=None	)   ->	List[Any]:



            '''simple docstring'''




            __snake_case            =	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
                 return output

            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE = None	)   ->	List[int]:



            '''simple docstring'''




            __snake_case            =	[self.sep_token_id]
            __snake_case            =	[self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep	) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]




       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE = None     ,	__SCREAMING_SNAKE_CASE = PaddingStrategy.DO_NOT_PAD     ,	__SCREAMING_SNAKE_CASE = None     ,	__SCREAMING_SNAKE_CASE = None     ,	)   ->	dict:



            '''simple docstring'''




            __snake_case            =	super()._pad(
                encoded_inputs=__SCREAMING_SNAKE_CASE     ,	max_length=__SCREAMING_SNAKE_CASE     ,	padding_strategy=__SCREAMING_SNAKE_CASE     ,	pad_to_multiple_of=__SCREAMING_SNAKE_CASE     ,	return_attention_mask=__SCREAMING_SNAKE_CASE     ,	)

            # Load from model defaults
            if return_attention_mask is None:
                 __snake_case            =	'''attention_mask''' in self.model_input_names

            if return_attention_mask and "global_attention_mask" in encoded_inputs:
                 __snake_case            =	encoded_inputs[self.model_input_names[0]]
                 # `global_attention_mask` need to have the same length as other (sequential) inputs.
                 __snake_case            =	len(encoded_inputs['''global_attention_mask''']	) != len(__SCREAMING_SNAKE_CASE	)

                 if needs_to_be_padded:
                      __snake_case            =	len(__SCREAMING_SNAKE_CASE	) - len(encoded_inputs['''global_attention_mask''']	)

                      if self.padding_side == "right":
                           # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                           __snake_case            =	(
                               encoded_inputs['''global_attention_mask'''] + [-1] * difference
                           )
                      elif self.padding_side == "left":
                           __snake_case            =	[-1] * difference + encoded_inputs[
                               '''global_attention_mask'''
                           ]
                      else:
                           raise ValueError('''Invalid padding strategy:''' + str(self.padding_side	)	)

            return encoded_inputs






