


"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import numpy
		import tensorflow as tf

		from transformers import (
		    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    BertConfig,
		    DPRConfig,
		    TFDPRContextEncoder,
		    TFDPRQuestionEncoder,
		    TFDPRReader,
		)







class    __a						:







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase=13      ,	UpperCAmelCase=7      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=True      ,	UpperCAmelCase=99      ,	UpperCAmelCase=32      ,	UpperCAmelCase=2      ,	UpperCAmelCase=4      ,	UpperCAmelCase=37      ,	UpperCAmelCase="gelu"      ,	UpperCAmelCase=0.1      ,	UpperCAmelCase=0.1      ,	UpperCAmelCase=512      ,	UpperCAmelCase=16      ,	UpperCAmelCase=2      ,	UpperCAmelCase=0.0_2      ,	UpperCAmelCase=3      ,	UpperCAmelCase=4      ,	UpperCAmelCase=None      ,	UpperCAmelCase=0      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			parent
					lowerCAmelCase_      							=			batch_size
					lowerCAmelCase_      							=			seq_length
					lowerCAmelCase_      							=			is_training
					lowerCAmelCase_      							=			use_input_mask
					lowerCAmelCase_      							=			use_token_type_ids
					lowerCAmelCase_      							=			use_labels
					lowerCAmelCase_      							=			vocab_size
					lowerCAmelCase_      							=			hidden_size
					lowerCAmelCase_      							=			num_hidden_layers
					lowerCAmelCase_      							=			num_attention_heads
					lowerCAmelCase_      							=			intermediate_size
					lowerCAmelCase_      							=			hidden_act
					lowerCAmelCase_      							=			hidden_dropout_prob
					lowerCAmelCase_      							=			attention_probs_dropout_prob
					lowerCAmelCase_      							=			max_position_embeddings
					lowerCAmelCase_      							=			type_vocab_size
					lowerCAmelCase_      							=			type_sequence_label_size
					lowerCAmelCase_      							=			initializer_range
					lowerCAmelCase_      							=			num_labels
					lowerCAmelCase_      							=			num_choices
					lowerCAmelCase_      							=			scope
					lowerCAmelCase_      							=			projection_dim







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.vocab_size   )

					lowerCAmelCase_      							=			None
					if self.use_input_mask:
						# follow test_modeling_tf_ctrl.py
						lowerCAmelCase_      							=			random_attention_mask([self.batch_size, self.seq_length]   )

					lowerCAmelCase_      							=			None
					if self.use_token_type_ids:
						lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.type_vocab_size   )

					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			None
					if self.use_labels:
						lowerCAmelCase_      							=			ids_tensor([self.batch_size]      ,	self.type_sequence_label_size   )
						lowerCAmelCase_      							=			ids_tensor([self.batch_size, self.seq_length]      ,	self.num_labels   )
						lowerCAmelCase_      							=			ids_tensor([self.batch_size]      ,	self.num_choices   )

					lowerCAmelCase_      							=			BertConfig(
					    vocab_size=self.vocab_size      ,	hidden_size=self.hidden_size      ,	num_hidden_layers=self.num_hidden_layers      ,	num_attention_heads=self.num_attention_heads      ,	intermediate_size=self.intermediate_size      ,	hidden_act=self.hidden_act      ,	hidden_dropout_prob=self.hidden_dropout_prob      ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,	max_position_embeddings=self.max_position_embeddings      ,	type_vocab_size=self.type_vocab_size      ,	is_decoder=a_      ,	initializer_range=self.initializer_range      ,	)
					lowerCAmelCase_      							=			DPRConfig(projection_dim=self.projection_dim      ,	**config.to_dict()   )

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			TFDPRContextEncoder(config=a_   )
					lowerCAmelCase_      							=			model(a_      ,	attention_mask=a_      ,	token_type_ids=a_   )
					lowerCAmelCase_      							=			model(a_      ,	token_type_ids=a_   )
					lowerCAmelCase_      							=			model(a_   )
					self.parent.assertEqual(result.pooler_output.shape      ,	(self.batch_size, self.projection_dim or self.hidden_size)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			TFDPRQuestionEncoder(config=a_   )
					lowerCAmelCase_      							=			model(a_      ,	attention_mask=a_      ,	token_type_ids=a_   )
					lowerCAmelCase_      							=			model(a_      ,	token_type_ids=a_   )
					lowerCAmelCase_      							=			model(a_   )
					self.parent.assertEqual(result.pooler_output.shape      ,	(self.batch_size, self.projection_dim or self.hidden_size)   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			TFDPRReader(config=a_   )
					lowerCAmelCase_      							=			model(a_      ,	attention_mask=a_   )

					self.parent.assertEqual(result.start_logits.shape      ,	(self.batch_size, self.seq_length)   )
					self.parent.assertEqual(result.end_logits.shape      ,	(self.batch_size, self.seq_length)   )
					self.parent.assertEqual(result.relevance_logits.shape      ,	(self.batch_size,)   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.prepare_config_and_inputs()
					(
					    lowerCAmelCase_  
					)      							=			config_and_inputs
					lowerCAmelCase_      							=			{'input_ids': input_ids}
					return config, inputs_dict







@require_tf
class    __a						( __snake_case					,			__snake_case					,			unittest.TestCase							):
				lowerCamelCase     :       List[str]			       =(
				    (
				        TFDPRContextEncoder,
				        TFDPRQuestionEncoder,
				        TFDPRReader,
				    )
				    if is_tf_available()
				    else ()
				)
				lowerCamelCase     :       List[Any]			       ={"""feature-extraction""": TFDPRQuestionEncoder} if is_tf_available() else {}

				lowerCamelCase     :       Optional[int]			       =False
				lowerCamelCase     :       Dict			       =False
				lowerCamelCase     :       Optional[Any]			       =False
				lowerCamelCase     :       str			       =False
				lowerCamelCase     :       int			       =False







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			TFDPRModelTester(self   )
					lowerCAmelCase_      							=			ConfigTester(self      ,	config_class=a_      ,	hidden_size=37   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					self.config_tester.run_common_tests()







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_dpr_context_encoder(*a_   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_dpr_question_encoder(*a_   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_dpr_reader(*a_   )







				@slow
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowerCAmelCase_      							=			TFDPRContextEncoder.from_pretrained(a_   )
						self.assertIsNotNone(a_   )

					for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowerCAmelCase_      							=			TFDPRContextEncoder.from_pretrained(a_   )
						self.assertIsNotNone(a_   )

					for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowerCAmelCase_      							=			TFDPRQuestionEncoder.from_pretrained(a_   )
						self.assertIsNotNone(a_   )

					for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowerCAmelCase_      							=			TFDPRReader.from_pretrained(a_   )
						self.assertIsNotNone(a_   )







@require_tf
class    __a						( unittest.TestCase							):







				@slow
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			TFDPRQuestionEncoder.from_pretrained('''facebook/dpr-question_encoder-single-nq-base'''   )

					lowerCAmelCase_      							=			tf.constant(
					    [[101, 7592, 1010, 2003, 2026, 3899, 1_0140, 1029, 102]]   )  # [CLS] hello, is my dog cute? [SEP]
					lowerCAmelCase_      							=			model(a_   )[0]  # embedding shape = (1, 768)
					# compare the actual values for a slice.
					lowerCAmelCase_      							=			tf.constant(
					    [
					        [
					            0.0_3_2_3_6_2_5_3,
					            0.1_2_7_5_3_3_3_5,
					            0.1_6_8_1_8_5_0_9,
					            0.0_0_2_7_9_7_8_6,
					            0.3_8_9_6_9_3_3,
					            0.2_4_2_6_4_9_4_5,
					            0.2_1_7_8_9_7_1,
					            -0.0_2_3_3_5_2_2_7,
					            -0.0_8_4_8_1_9_5_9,
					            -0.1_4_3_2_4_1_1_7,
					        ]
					    ]   )
					self.assertTrue(numpy.allclose(output[:, :10].numpy()      ,	expected_slice.numpy()      ,	atol=1E-4   )   )




'''simple docstring'''

import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						a_		(     snake_case    ,   unittest.TestCase    ):
							UpperCAmelCase			:		int        =				MgpstrTokenizer
							UpperCAmelCase			:		List[Any]        =				False
							UpperCAmelCase			:		Optional[int]        =				{}
							UpperCAmelCase			:		Any        =				False


							def    UpperCamelCase     (				self   :						Any			)			->					Dict:
										super().setUp()

										# fmt: off
										snake_case:    List[Any]      							=['[GO]', '[s]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
										# fmt: on
										snake_case:    Union[str, Any]      							=dict(zip(a_      ,						range(len(a_			)			)			)			)

										snake_case:    List[Any]      							=os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES['vocab_file']			)
										with open(self.vocab_file      ,						'w'      ,						encoding='utf-8'			) as fp:
													fp.write(json.dumps(a_			) + '\n'			)


							def    UpperCamelCase     (				self   :						Optional[Any]      ,						**a_   :						List[Any]			)			->					str:
										return MgpstrTokenizer.from_pretrained(self.tmpdirname      ,						**a_			)


							def    UpperCamelCase     (				self   :						int      ,						a_   :						Dict			)			->					int:
										snake_case:    Any      							='tester'
										snake_case:    List[str]      							='tester'
										return input_text, output_text


							@unittest.skip('MGP-STR always lower cases letters.'			)
							def    UpperCamelCase     (				self   :						Any			)			->					Dict:
										pass


							def    UpperCamelCase     (				self   :						Optional[int]			)			->					int:
										snake_case:    int      							=self.get_tokenizers(do_lower_case=a_			)
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''			):
																snake_case:    Optional[Any]      							='[SPECIAL_TOKEN]'

																tokenizer.add_special_tokens({'cls_token': special_token}			)
																snake_case:    List[str]      							=tokenizer.encode([special_token]      ,						add_special_tokens=a_			)
																self.assertEqual(len(a_			)      ,						1			)

																snake_case:    Union[str, Any]      							=tokenizer.decode(a_      ,						skip_special_tokens=a_			)
																self.assertTrue(special_token not in decoded			)


							def    UpperCamelCase     (				self   :						int			)			->					str:
										snake_case:    int      							=self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(F'''{tokenizer.__class__.__name__}'''			):
																snake_case							,  snake_case:    Dict      							=self.get_input_output_texts(a_			)

																snake_case:    Dict      							=tokenizer.tokenize(a_			)
																snake_case:    str      							=tokenizer.convert_tokens_to_ids(a_			)
																snake_case:    List[Any]      							=tokenizer.encode(a_      ,						add_special_tokens=a_			)
																self.assertListEqual(a_      ,						a_			)

																snake_case:    str      							=tokenizer.convert_ids_to_tokens(a_			)
																self.assertNotEqual(len(a_			)      ,						0			)
																snake_case:    Tuple      							=tokenizer.decode(a_			)
																self.assertIsInstance(a_      ,						a_			)

																self.assertEqual(text_a.replace(' '      ,						''			)      ,						a_			)


							@unittest.skip('MGP-STR tokenizer only handles one sequence.'			)
							def    UpperCamelCase     (				self   :						Dict			)			->					str:
										pass




							@unittest.skip('inputs cannot be pretokenized in MgpstrTokenizer'			)
							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					Optional[int]:
										pass




