






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import numpy
	import tensorflow as tf

	from transformers import (
	    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    BertConfig,
	    DPRConfig,
	    TFDPRContextEncoder,
	    TFDPRQuestionEncoder,
	    TFDPRReader,
	)




class 					UpperCamelCase_ :



	'''simple docstring'''




	def __init__(			self			,     a			,     a=13			,     a=7			,     a=True			,     a=True			,     a=True			,     a=True			,     a=99			,     a=32			,     a=2			,     a=4			,     a=37			,     a="gelu"			,     a=0.1			,     a=0.1			,     a=5_12			,     a=16			,     a=2			,     a=0.02			,     a=3			,     a=4			,     a=None			,     a=0			,     )			->       Optional[int]:
								snake_case_           =				parent
								snake_case_           =				batch_size
								snake_case_           =				seq_length
								snake_case_           =				is_training
								snake_case_           =				use_input_mask
								snake_case_           =				use_token_type_ids
								snake_case_           =				use_labels
								snake_case_           =				vocab_size
								snake_case_           =				hidden_size
								snake_case_           =				num_hidden_layers
								snake_case_           =				num_attention_heads
								snake_case_           =				intermediate_size
								snake_case_           =				hidden_act
								snake_case_           =				hidden_dropout_prob
								snake_case_           =				attention_probs_dropout_prob
								snake_case_           =				max_position_embeddings
								snake_case_           =				type_vocab_size
								snake_case_           =				type_sequence_label_size
								snake_case_           =				initializer_range
								snake_case_           =				num_labels
								snake_case_           =				num_choices
								snake_case_           =				scope
								snake_case_           =				projection_dim




	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								snake_case_           =				ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size   )

								snake_case_           =				None
								if self.use_input_mask:
															# follow test_modeling_tf_ctrl.py
															snake_case_           =				random_attention_mask([self.batch_size, self.seq_length]   )

								snake_case_           =				None
								if self.use_token_type_ids:
															snake_case_           =				ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size   )

								snake_case_           =				None
								snake_case_           =				None
								snake_case_           =				None
								if self.use_labels:
															snake_case_           =				ids_tensor([self.batch_size]			,     self.type_sequence_label_size   )
															snake_case_           =				ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels   )
															snake_case_           =				ids_tensor([self.batch_size]			,     self.num_choices   )

								snake_case_           =				BertConfig(
								    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     is_decoder=a			,     initializer_range=self.initializer_range			,     )
								snake_case_           =				DPRConfig(projection_dim=self.projection_dim			,     **config.to_dict()   )

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




	def 				_UpperCamelCase     (			self			,     a			,     a			,     a			,     a			,     a			,     a			,     a   )			->       Tuple:
								snake_case_           =				TFDPRContextEncoder(config=a   )
								snake_case_           =				model(a			,     attention_mask=a			,     token_type_ids=a   )
								snake_case_           =				model(a			,     token_type_ids=a   )
								snake_case_           =				model(a   )
								self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.projection_dim or self.hidden_size)   )




	def 				_UpperCamelCase     (			self			,     a			,     a			,     a			,     a			,     a			,     a			,     a   )			->       Tuple:
								snake_case_           =				TFDPRQuestionEncoder(config=a   )
								snake_case_           =				model(a			,     attention_mask=a			,     token_type_ids=a   )
								snake_case_           =				model(a			,     token_type_ids=a   )
								snake_case_           =				model(a   )
								self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.projection_dim or self.hidden_size)   )




	def 				_UpperCamelCase     (			self			,     a			,     a			,     a			,     a			,     a			,     a			,     a   )			->       Optional[Any]:
								snake_case_           =				TFDPRReader(config=a   )
								snake_case_           =				model(a			,     attention_mask=a   )

								self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)   )
								self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)   )
								self.parent.assertEqual(result.relevance_logits.shape			,     (self.batch_size,)   )






	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								snake_case_           =				self.prepare_config_and_inputs()
								(
								    (
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						(
								    snake_case_
								)     ,						
								)           =				config_and_inputs
								snake_case_           =				{'input_ids': input_ids}
								return config, inputs_dict






@require_tf
class 					UpperCamelCase_ (     snake_case_			,   snake_case_			,   unittest.TestCase							):



	'''simple docstring'''




	lowerCAmelCase						=					(
	    (
	        TFDPRContextEncoder,
	        TFDPRQuestionEncoder,
	        TFDPRReader,
	    )
	    if is_tf_available()
	    else ()
	)
	lowerCAmelCase						=					{'''feature-extraction''': TFDPRQuestionEncoder} if is_tf_available() else {}

	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False




	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								snake_case_           =				TFDPRModelTester(self   )
								snake_case_           =				ConfigTester(self			,     config_class=a			,     hidden_size=37   )




	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								self.config_tester.run_common_tests()




	def 				_UpperCamelCase     (			self   )			->       List[str]:
								snake_case_           =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_dpr_context_encoder(*a   )




	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								snake_case_           =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_dpr_question_encoder(*a   )




	def 				_UpperCamelCase     (			self   )			->       Tuple:
								snake_case_           =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_dpr_reader(*a   )






	@slow
	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case_           =				TFDPRContextEncoder.from_pretrained(a   )
															self.assertIsNotNone(a   )

								for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case_           =				TFDPRContextEncoder.from_pretrained(a   )
															self.assertIsNotNone(a   )

								for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case_           =				TFDPRQuestionEncoder.from_pretrained(a   )
															self.assertIsNotNone(a   )

								for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case_           =				TFDPRReader.from_pretrained(a   )
															self.assertIsNotNone(a   )






@require_tf
class 					UpperCamelCase_ (     unittest.TestCase							):



	'''simple docstring'''




	@slow
	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								snake_case_           =				TFDPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'   )

								snake_case_           =				tf.constant(
								    [[1_01, 75_92, 10_10, 20_03, 20_26, 38_99, 1_01_40, 10_29, 1_02]]   )  # [CLS] hello, is my dog cute? [SEP]
								snake_case_           =				model(a   )[0]  # embedding shape = (1, 768)
								# compare the actual values for a slice.
								snake_case_           =				tf.constant(
								    [
								        [
								            0.03_236_253,
								            0.12_753_335,
								            0.16_818_509,
								            0.00_279_786,
								            0.3_896_933,
								            0.24_264_945,
								            0.2_178_971,
								            -0.02_335_227,
								            -0.08_481_959,
								            -0.14_324_117,
								        ]
								    ]   )
								self.assertTrue(numpy.allclose(output[:, :10].numpy()			,     expected_slice.numpy()			,     atol=1E-4   )   )













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_rembert import RemBertTokenizer
else:
	lowercase    				=   None

lowercase    				=   logging.get_logger(__name__)
lowercase    				=   {"vocab_file": "sentencepiece.model", "tokenizer_file": "tokenizer.json"}

lowercase    				=   {
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
    "tokenizer_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/tokenizer.json",
    },
}

lowercase    				=   {
    "google/rembert": 256,
}

lowercase    				=   "▁"




class 					UpperCamelCase_ (     snake_case_							):



	'''simple docstring'''




	lowerCAmelCase						=					VOCAB_FILES_NAMES
	lowerCAmelCase						=					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase						=					RemBertTokenizer




	def __init__(			self			,     a=None			,     a=None			,     a=True			,     a=True			,     a=False			,     a="[CLS]"			,     a="[SEP]"			,     a="<unk>"			,     a="[SEP]"			,     a="<pad>"			,     a="[CLS]"			,     a="[MASK]"			,     **a			,     )			->       Union[str, Any]:
								# Mask token behave like a normal word, i.e. include the space before it
								snake_case_           =				AddedToken(a			,     lstrip=a			,     rstrip=a   ) if isinstance(a			,     a   ) else mask_token

								super().__init__(
								    a			,     tokenizer_file=a			,     do_lower_case=a			,     remove_space=a			,     keep_accents=a			,     bos_token=a			,     eos_token=a			,     unk_token=a			,     sep_token=a			,     pad_token=a			,     cls_token=a			,     mask_token=a			,     **a			,     )

								snake_case_           =				do_lower_case
								snake_case_           =				remove_space
								snake_case_           =				keep_accents
								snake_case_           =				vocab_file
								snake_case_           =				False if not self.vocab_file else True




	def 				_UpperCamelCase     (			self			,     a			,     a = None   )			->       List[int]:
								snake_case_           =				[self.sep_token_id]
								snake_case_           =				[self.cls_token_id]
								if token_ids_a is None:
															return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep




	def 				_UpperCamelCase     (			self			,     a			,     a = None			,     a = False   )			->       List[int]:

								if already_has_special_tokens:
															if token_ids_a is not None:
																						raise ValueError(
																						    'You should not supply a second sequence if the provided sequence of '
																						    'ids is already formatted with special tokens for the model.'   )
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
															return [1] + ([0] * len(a   )) + [1] + ([0] * len(a   )) + [1]
								return [1] + ([0] * len(a   )) + [1]




	def 				_UpperCamelCase     (			self			,     a			,     a = None   )			->       List[int]:
								snake_case_           =				[self.sep_token_id]
								snake_case_           =				[self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep   ) * [0]
								return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






	def 				_UpperCamelCase     (			self			,     a			,     a = None   )			->       Tuple[str]:
								if not os.path.isdir(a   ):
															logger.error('Vocabulary path ({}) should be a directory'.format(a   )   )
															return
								snake_case_           =				os.path.join(
								    a			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

								if os.path.abspath(self.vocab_file   ) != os.path.abspath(a   ):
															copyfile(self.vocab_file			,     a   )

								return (out_vocab_file,)





