





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel





def        _lowercase   (  _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase=None     , _UpperCAmelCase=None			)  ->					Dict:
				if attention_mask is None:
								lowerCamelCase			=tf.cast(tf.math.not_equal(_UpperCAmelCase     , config.pad_token_id			)     , tf.inta			)
				return {"input_ids": input_ids, "attention_mask": attention_mask}

@require_tf
class  __A       :
				__A								=			OPTConfig
				__A								=			{}
				__A								=			"""gelu"""


				def __init__(						self				,	UpperCAmelCase_				,	UpperCAmelCase_=13				,	UpperCAmelCase_=7				,	UpperCAmelCase_=True				,	UpperCAmelCase_=False				,	UpperCAmelCase_=99				,	UpperCAmelCase_=16				,	UpperCAmelCase_=2				,	UpperCAmelCase_=4				,	UpperCAmelCase_=4				,	UpperCAmelCase_="gelu"				,	UpperCAmelCase_=0.1				,	UpperCAmelCase_=0.1				,	UpperCAmelCase_=20				,	UpperCAmelCase_=2				,	UpperCAmelCase_=1				,	UpperCAmelCase_=0				,	UpperCAmelCase_=16				,	UpperCAmelCase_=16				,	):
								lowerCamelCase			=parent
								lowerCamelCase			=batch_size
								lowerCamelCase			=seq_length
								lowerCamelCase			=is_training
								lowerCamelCase			=use_labels
								lowerCamelCase			=vocab_size
								lowerCamelCase			=hidden_size
								lowerCamelCase			=num_hidden_layers
								lowerCamelCase			=num_attention_heads
								lowerCamelCase			=intermediate_size
								lowerCamelCase			=hidden_act
								lowerCamelCase			=hidden_dropout_prob
								lowerCamelCase			=attention_probs_dropout_prob
								lowerCamelCase			=max_position_embeddings
								lowerCamelCase			=eos_token_id
								lowerCamelCase			=pad_token_id
								lowerCamelCase			=bos_token_id
								lowerCamelCase			=embed_dim
								lowerCamelCase			=word_embed_proj_dim
								lowerCamelCase			=False


				def _snake_case    (						self							):
								lowerCamelCase			=ids_tensor([self.batch_size, self.seq_length - 1]				,	self.vocab_size							)
								lowerCamelCase			=tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)				,	1							)
								lowerCamelCase			=tf.concat([input_ids, eos_tensor]				,	axis=1							)

								lowerCamelCase			=self.config_cls(
								    vocab_size=self.vocab_size				,	hidden_size=self.hidden_size				,	num_hidden_layers=self.num_hidden_layers				,	num_attention_heads=self.num_attention_heads				,	ffn_dim=self.intermediate_size				,	dropout=self.hidden_dropout_prob				,	attention_dropout=self.attention_probs_dropout_prob				,	max_position_embeddings=self.max_position_embeddings				,	eos_token_id=self.eos_token_id				,	bos_token_id=self.bos_token_id				,	pad_token_id=self.pad_token_id				,	embed_dim=self.embed_dim				,	word_embed_proj_dim=self.word_embed_proj_dim				,	is_encoder_decoder=UpperCAmelCase_				,	**self.config_updates				,	)
								lowerCamelCase			=prepare_opt_inputs_dict(UpperCAmelCase_				,	UpperCAmelCase_							)
								return config, inputs_dict


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=TFOPTModel(config=UpperCAmelCase_							)
								lowerCamelCase			=inputs_dict["""input_ids"""]

								lowerCamelCase			=input_ids[:1, :]
								lowerCamelCase			=inputs_dict["""attention_mask"""][:1, :]
								lowerCamelCase			=1

								# first forward pass
								lowerCamelCase			=model(UpperCAmelCase_				,	attention_mask=UpperCAmelCase_				,	use_cache=UpperCAmelCase_							)

								lowerCamelCase	,					lowerCamelCase			=outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								lowerCamelCase			=ids_tensor((self.batch_size, 3)				,	config.vocab_size							)
								lowerCamelCase			=tf.cast(ids_tensor((self.batch_size, 3)				,	2							)				,	tf.inta							)

								# append to next input_ids and
								lowerCamelCase			=tf.concat([input_ids, next_tokens]				,	axis=-1							)
								lowerCamelCase			=tf.concat([attention_mask, next_attn_mask]				,	axis=-1							)

								lowerCamelCase			=model(UpperCAmelCase_				,	attention_mask=UpperCAmelCase_							)[0]
								lowerCamelCase			=model(UpperCAmelCase_				,	attention_mask=UpperCAmelCase_				,	past_key_values=UpperCAmelCase_							)[0]

								self.parent.assertEqual(next_tokens.shape[1]				,	output_from_past.shape[1]							)

								# select random slice
								lowerCamelCase			=int(ids_tensor((1,)				,	output_from_past.shape[-1]							)							)
								lowerCamelCase			=output_from_no_past[:, -3:, random_slice_idx]
								lowerCamelCase			=output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(UpperCAmelCase_				,	UpperCAmelCase_				,	rtol=1E-3							)




@require_tf
class  __A       (  a      ,   a      ,   unittest.TestCase      ):
				__A								=			(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
				__A								=			(TFOPTForCausalLM,) if is_tf_available() else ()
				__A								=			(
				    {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
				)
				__A								=			False
				__A								=			False
				__A								=			False
				__A								=			10


				def _snake_case    (						self							):
								lowerCamelCase			=TFOPTModelTester(self							)
								lowerCamelCase			=ConfigTester(self				,	config_class=UpperCAmelCase_							)


				def _snake_case    (						self							):
								self.config_tester.run_common_tests()


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*UpperCAmelCase_							)


				def _snake_case    (						self							):
								lowerCamelCase	,					lowerCamelCase			=self.model_tester.prepare_config_and_inputs_for_common()

								def _get_word_embedding_weight(UpperCAmelCase_				,	UpperCAmelCase_							):
												if hasattr(UpperCAmelCase_				,	"""weight"""							):
																return embedding_layer.weight
												else:
																# Here we build the word embeddings weights if not exists.
																# And then we retry to get the attribute once built.
																model.build()
																if hasattr(UpperCAmelCase_				,	"""weight"""							):
																				return embedding_layer.weight
																else:
																				return None

								for model_class in self.all_model_classes:
												for size in [config.vocab_size - 10, config.vocab_size + 10]:
																# build the embeddings
																lowerCamelCase			=model_class(config=UpperCAmelCase_							)
																lowerCamelCase			=_get_word_embedding_weight(UpperCAmelCase_				,	model.get_input_embeddings()							)
																lowerCamelCase			=_get_word_embedding_weight(UpperCAmelCase_				,	model.get_output_embeddings()							)

																# reshape the embeddings
																model.resize_token_embeddings(UpperCAmelCase_							)
																lowerCamelCase			=_get_word_embedding_weight(UpperCAmelCase_				,	model.get_input_embeddings()							)
																lowerCamelCase			=_get_word_embedding_weight(UpperCAmelCase_				,	model.get_output_embeddings()							)

																# check that the resized embeddings size matches the desired size.
																lowerCamelCase			=size if size is not None else config.vocab_size

																self.assertEqual(new_input_embeddings.shape[0]				,	UpperCAmelCase_							)

																# check that weights remain the same after resizing
																lowerCamelCase			=True
																for pa, pa in zip(old_input_embeddings.value()				,	new_input_embeddings.value()							):
																				if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
																								lowerCamelCase			=False
																self.assertTrue(UpperCAmelCase_							)

																if old_output_embeddings is not None and new_output_embeddings is not None:
																				self.assertEqual(new_output_embeddings.shape[0]				,	UpperCAmelCase_							)

																				lowerCamelCase			=True
																				for pa, pa in zip(old_output_embeddings.value()				,	new_output_embeddings.value()							):
																								if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
																												lowerCamelCase			=False
																				self.assertTrue(UpperCAmelCase_							)





def        _lowercase   (  _UpperCAmelCase			)  ->					int:
				return tf.constant(_UpperCAmelCase     , dtype=tf.intaa			)

@require_tf
class  __A       (  unittest.TestCase      ):
				__A								=			99


				def _snake_case    (						self							):
								lowerCamelCase			=tf.ones((4, 1)				,	dtype=tf.intaa							) * 2
								lowerCamelCase			=tf.concat([ids_tensor((4, 6)				,	self.vocab_size - 3							) + 3, eos_column_vector]				,	axis=1							)
								lowerCamelCase			=input_ids.shape[0]
								lowerCamelCase			=OPTConfig(
								    vocab_size=self.vocab_size				,	hidden_size=24				,	num_hidden_layers=2				,	num_attention_heads=2				,	ffn_dim=32				,	max_position_embeddings=48				,	eos_token_id=2				,	pad_token_id=1				,	bos_token_id=0				,	)
								return config, input_ids, batch_size




@require_sentencepiece
@require_tf
class  __A       (  unittest.TestCase      ):


				@slow
				def _snake_case    (						self							):
								lowerCamelCase			=TFOPTModel.from_pretrained("""facebook/opt-350m"""							)
								lowerCamelCase			=_long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]							)
								lowerCamelCase			=tf.not_equal(UpperCAmelCase_				,	model.config.pad_token_id							)
								with tf.GradientTape():
												lowerCamelCase			=model(input_ids=UpperCAmelCase_				,	attention_mask=UpperCAmelCase_							).last_hidden_state
								lowerCamelCase			=(1, 11, 512)
								self.assertEqual(output.shape				,	UpperCAmelCase_							)
								lowerCamelCase			=tf.constant(
								    [[-0.2_8_7_3, -1.9_2_1_8, -0.3_0_3_3], [-1.2_7_1_0, -0.1_3_3_8, -0.1_9_0_2], [0.4_0_9_5, 0.1_2_1_4, -1.3_1_2_1]]							)
								self.assertTrue(np.allclose(output[:, :3, :3]				,	UpperCAmelCase_				,	atol=4E-3							)							)

								lowerCamelCase			=tf.function(UpperCAmelCase_				,	jit_compile=UpperCAmelCase_							)
								lowerCamelCase			=xla_generate(UpperCAmelCase_				,	UpperCAmelCase_							)[0]
								self.assertTrue(np.allclose(output[:, :3, :3]				,	UpperCAmelCase_				,	atol=4E-2							)							)




@require_tf
@slow
class  __A       (  unittest.TestCase      ):


				def _snake_case    (						self							):
								super().setUp()
								lowerCamelCase			="""facebook/opt-350m"""


				def _snake_case    (						self							):
								lowerCamelCase			=TFOPTForCausalLM.from_pretrained(self.path_model							)
								lowerCamelCase			=GPTaTokenizer.from_pretrained(self.path_model							)

								lowerCamelCase			=[
								    """Today is a beautiful day and I want to""",
								    """In the city of""",
								    """Paris is the capital of France and""",
								    """Computers and mobile phones have taken""",
								]
								# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
								lowerCamelCase			=tokenizer(UpperCAmelCase_				,	return_tensors="""tf"""				,	padding=UpperCAmelCase_				,	add_special_tokens=UpperCAmelCase_							)
								lowerCamelCase			=tf.math.reduce_mean(model(inputs.input_ids				,	attention_mask=inputs.attention_mask							)[0]				,	axis=-1							)
								lowerCamelCase			=tf.constant(
								    [
								        [1.3_8_5_1, -13.8923, -10.5229, -10.7533, -0.2_3_0_9, -10.2384, -0.5_3_6_5, -9.0_9_4_7, -5.1_6_7_0],
								        [-4.7_0_7_3, -10.6276, -3.9_4_1_5, -21.5242, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2],
								        [0.6_2_4_7, -3.4_2_2_9, -8.9_1_7_9, -1.4_2_9_7, -14.1650, 1.4_1_4_6, -9.0_2_1_8, -0.2_7_0_3, -0.2_7_0_3],
								        [6.4_7_8_3, -1.9_9_1_3, -10.7926, -2.3_3_3_6, 1.5_0_9_2, -0.9_9_7_4, -6.8_2_1_3, 1.3_4_7_7, 1.3_4_7_7],
								    ]							)
								self.assertTrue(np.allclose(UpperCAmelCase_				,	UpperCAmelCase_				,	atol=1E-4							)							)

								lowerCamelCase			=tf.function(UpperCAmelCase_				,	jit_compile=UpperCAmelCase_							)
								lowerCamelCase			=tf.math.reduce_mean(xla_generate(inputs.input_ids				,	attention_mask=inputs.attention_mask							)[0]				,	axis=-1							)
								self.assertTrue(np.allclose(UpperCAmelCase_				,	UpperCAmelCase_				,	atol=1E-4							)							)




@require_tf
@slow
class  __A       (  unittest.TestCase      ):


				@property
				def _snake_case    (						self							):
								return [
								    "Today is a beautiful day and I want",
								    "In the city of",
								    "Paris is the capital of France and",
								    "Computers and mobile phones have taken",
								]


				def _snake_case    (						self							):
								lowerCamelCase			="""facebook/opt-125m"""

								lowerCamelCase			=[
								    """Today is a beautiful day and I want to""",
								    """In the city of New York, the city""",
								    """Paris is the capital of France and the capital""",
								    """Computers and mobile phones have taken over the""",
								]

								lowerCamelCase			=[]
								lowerCamelCase			=GPTaTokenizer.from_pretrained(UpperCAmelCase_							)
								lowerCamelCase			=TFOPTForCausalLM.from_pretrained(UpperCAmelCase_							)

								for prompt in self.prompts:
												lowerCamelCase			=tokenizer(UpperCAmelCase_				,	return_tensors="""tf"""							).input_ids

												lowerCamelCase			=model.generate(UpperCAmelCase_				,	max_length=10							)

												lowerCamelCase			=tokenizer.batch_decode(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)
												predicted_outputs += generated_string

								self.assertListEqual(UpperCAmelCase_				,	UpperCAmelCase_							)


				def _snake_case    (						self							):
								lowerCamelCase			="""facebook/opt-350m"""

								lowerCamelCase			=GPTaTokenizer.from_pretrained(UpperCAmelCase_							)
								lowerCamelCase			=TFOPTForCausalLM.from_pretrained(UpperCAmelCase_							)

								lowerCamelCase			="""left"""

								# use different length sentences to test batching
								lowerCamelCase			=[
								    """Hello, my dog is a little""",
								    """Today, I""",
								]

								lowerCamelCase			=tokenizer(UpperCAmelCase_				,	return_tensors="""tf"""				,	padding=UpperCAmelCase_							)
								lowerCamelCase			=inputs["""input_ids"""]

								lowerCamelCase			=model.generate(input_ids=UpperCAmelCase_				,	attention_mask=inputs["""attention_mask"""]							)

								lowerCamelCase			=tokenizer(sentences[0]				,	return_tensors="""tf"""							).input_ids
								lowerCamelCase			=model.generate(input_ids=UpperCAmelCase_							)

								lowerCamelCase			=inputs_non_padded.shape[-1] - tf.math.reduce_sum(
								    tf.cast(inputs["""attention_mask"""][-1]				,	tf.intaa							)							)
								lowerCamelCase			=tokenizer(sentences[1]				,	return_tensors="""tf"""							).input_ids
								lowerCamelCase			=model.generate(input_ids=UpperCAmelCase_				,	max_length=model.config.max_length - num_paddings							)

								lowerCamelCase			=tokenizer.batch_decode(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)
								lowerCamelCase			=tokenizer.decode(output_non_padded[0]				,	skip_special_tokens=UpperCAmelCase_							)
								lowerCamelCase			=tokenizer.decode(output_padded[0]				,	skip_special_tokens=UpperCAmelCase_							)

								lowerCamelCase			=[
								    """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
								    """Today, I was in the middle of a conversation with a friend about the""",
								]
								self.assertListEqual(UpperCAmelCase_				,	UpperCAmelCase_							)
								self.assertListEqual(UpperCAmelCase_				,	[non_padded_sentence, padded_sentence]							)


				def _snake_case    (						self							):
								lowerCamelCase			="""facebook/opt-350m"""

								lowerCamelCase			=[
								    """Today is a beautiful day and I want to""",
								    """In the city of San Francisco, the city""",
								    """Paris is the capital of France and the capital""",
								    """Computers and mobile phones have taken over the""",
								]

								lowerCamelCase			=[]
								lowerCamelCase			=GPTaTokenizer.from_pretrained(UpperCAmelCase_							)
								lowerCamelCase			=TFOPTForCausalLM.from_pretrained(UpperCAmelCase_							)

								for prompt in self.prompts:
												lowerCamelCase			=tokenizer(UpperCAmelCase_				,	return_tensors="""tf"""							).input_ids

												lowerCamelCase			=model.generate(UpperCAmelCase_				,	max_length=10							)

												lowerCamelCase			=tokenizer.batch_decode(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)
												predicted_outputs += generated_string

								self.assertListEqual(UpperCAmelCase_				,	UpperCAmelCase_							)







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


UpperCAmelCase__  :     Optional[int]   			=logging.get_logger(__name__)

UpperCAmelCase__  :     int   			={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__  :     List[str]   			={
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

UpperCAmelCase__  :     Any   			={'''mobilebert-uncased''': 5_12}


UpperCAmelCase__  :     Any   			={}

class  __A       (  a      ):
				__A								=			VOCAB_FILES_NAMES
				__A								=			PRETRAINED_VOCAB_FILES_MAP
				__A								=			PRETRAINED_INIT_CONFIGURATION
				__A								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A								=			MobileBertTokenizer


				def __init__(						self				,	UpperCAmelCase_=None				,	UpperCAmelCase_=None				,	UpperCAmelCase_=True				,	UpperCAmelCase_="[UNK]"				,	UpperCAmelCase_="[SEP]"				,	UpperCAmelCase_="[PAD]"				,	UpperCAmelCase_="[CLS]"				,	UpperCAmelCase_="[MASK]"				,	UpperCAmelCase_=True				,	UpperCAmelCase_=None				,	**UpperCAmelCase_				,	):
								super().__init__(
								    UpperCAmelCase_				,	tokenizer_file=UpperCAmelCase_				,	do_lower_case=UpperCAmelCase_				,	unk_token=UpperCAmelCase_				,	sep_token=UpperCAmelCase_				,	pad_token=UpperCAmelCase_				,	cls_token=UpperCAmelCase_				,	mask_token=UpperCAmelCase_				,	tokenize_chinese_chars=UpperCAmelCase_				,	strip_accents=UpperCAmelCase_				,	**UpperCAmelCase_				,	)

								lowerCamelCase			=json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
								if (
								    normalizer_state.get("""lowercase"""				,	UpperCAmelCase_							) != do_lower_case
								    or normalizer_state.get("""strip_accents"""				,	UpperCAmelCase_							) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars"""				,	UpperCAmelCase_							) != tokenize_chinese_chars
								):
												lowerCamelCase			=getattr(UpperCAmelCase_				,	normalizer_state.pop("""type"""							)							)
												lowerCamelCase			=do_lower_case
												lowerCamelCase			=strip_accents
												lowerCamelCase			=tokenize_chinese_chars
												lowerCamelCase			=normalizer_class(**UpperCAmelCase_							)

								lowerCamelCase			=do_lower_case


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_=None							):
								lowerCamelCase			=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep							) * [0]
								return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=self._tokenizer.model.save(UpperCAmelCase_				,	name=UpperCAmelCase_							)
								return tuple(UpperCAmelCase_							)





