


'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import numpy
 import tensorflow as tf

 from transformers import (
     TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
     TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
     TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
     BertConfig,
     DPRConfig,
     TFDPRContextEncoder,
     TFDPRQuestionEncoder,
     TFDPRReader,
 )

class 						_lowerCamelCase					:




 '''simple docstring'''


 def __init__(      self      ,      __lowercase      ,      __lowercase=13      ,      __lowercase=7      ,      __lowercase=True      ,      __lowercase=True      ,      __lowercase=True      ,      __lowercase=True      ,      __lowercase=99      ,      __lowercase=32      ,      __lowercase=2      ,      __lowercase=4      ,      __lowercase=37      ,      __lowercase="gelu"      ,      __lowercase=0.1      ,      __lowercase=0.1      ,      __lowercase=512      ,      __lowercase=16      ,      __lowercase=2      ,      __lowercase=0.0_2      ,      __lowercase=3      ,      __lowercase=4      ,      __lowercase=None      ,      __lowercase=0      ,      ):

  """simple docstring"""







  __A  :		List[Any]       =			parent
  __A  :		Tuple       =			batch_size
  __A  :		Optional[int]       =			seq_length
  __A  :		Optional[Any]       =			is_training
  __A  :		Optional[Any]       =			use_input_mask
  __A  :		List[Any]       =			use_token_type_ids
  __A  :		Optional[Any]       =			use_labels
  __A  :		Optional[Any]       =			vocab_size
  __A  :		Union[str, Any]       =			hidden_size
  __A  :		Tuple       =			num_hidden_layers
  __A  :		Optional[Any]       =			num_attention_heads
  __A  :		Optional[int]       =			intermediate_size
  __A  :		Dict       =			hidden_act
  __A  :		int       =			hidden_dropout_prob
  __A  :		Optional[int]       =			attention_probs_dropout_prob
  __A  :		Tuple       =			max_position_embeddings
  __A  :		Tuple       =			type_vocab_size
  __A  :		List[str]       =			type_sequence_label_size
  __A  :		Union[str, Any]       =			initializer_range
  __A  :		Optional[int]       =			num_labels
  __A  :		List[Any]       =			num_choices
  __A  :		Any       =			scope
  __A  :		List[str]       =			projection_dim


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		int       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size						)

  __A  :		Tuple       =			None
  if self.use_input_mask:
   # follow test_modeling_tf_ctrl.py
   __A  :		str       =			random_attention_mask([self.batch_size, self.seq_length]						)

  __A  :		Union[str, Any]       =			None
  if self.use_token_type_ids:
   __A  :		int       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.type_vocab_size						)

  __A  :		str       =			None
  __A  :		Tuple       =			None
  __A  :		Union[str, Any]       =			None
  if self.use_labels:
   __A  :		int       =			ids_tensor([self.batch_size]      ,      self.type_sequence_label_size						)
   __A  :		Union[str, Any]       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels						)
   __A  :		Optional[int]       =			ids_tensor([self.batch_size]      ,      self.num_choices						)

  __A  :		Tuple       =			BertConfig(
      vocab_size=self.vocab_size      ,      hidden_size=self.hidden_size      ,      num_hidden_layers=self.num_hidden_layers      ,      num_attention_heads=self.num_attention_heads      ,      intermediate_size=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      max_position_embeddings=self.max_position_embeddings      ,      type_vocab_size=self.type_vocab_size      ,      is_decoder=__lowercase      ,      initializer_range=self.initializer_range      ,      )
  __A  :		Optional[Any]       =			DPRConfig(projection_dim=self.projection_dim      ,      **config.to_dict()						)

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  __A  :		Union[str, Any]       =			TFDPRContextEncoder(config=__lowercase						)
  __A  :		Any       =			model(__lowercase      ,      attention_mask=__lowercase      ,      token_type_ids=__lowercase						)
  __A  :		Optional[int]       =			model(__lowercase      ,      token_type_ids=__lowercase						)
  __A  :		List[str]       =			model(__lowercase						)
  self.parent.assertEqual(result.pooler_output.shape      ,      (self.batch_size, self.projection_dim or self.hidden_size)						)


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  __A  :		int       =			TFDPRQuestionEncoder(config=__lowercase						)
  __A  :		Tuple       =			model(__lowercase      ,      attention_mask=__lowercase      ,      token_type_ids=__lowercase						)
  __A  :		Optional[int]       =			model(__lowercase      ,      token_type_ids=__lowercase						)
  __A  :		Optional[Any]       =			model(__lowercase						)
  self.parent.assertEqual(result.pooler_output.shape      ,      (self.batch_size, self.projection_dim or self.hidden_size)						)


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  __A  :		Tuple       =			TFDPRReader(config=__lowercase						)
  __A  :		Optional[int]       =			model(__lowercase      ,      attention_mask=__lowercase						)

  self.parent.assertEqual(result.start_logits.shape      ,      (self.batch_size, self.seq_length)						)
  self.parent.assertEqual(result.end_logits.shape      ,      (self.batch_size, self.seq_length)						)
  self.parent.assertEqual(result.relevance_logits.shape      ,      (self.batch_size,)						)







 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		List[Any]       =			self.prepare_config_and_inputs()
  (
      (
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,
  )  :		int       =			config_and_inputs
  __A  :		Optional[int]       =			{'input_ids': input_ids}
  return config, inputs_dict


@require_tf
class 						_lowerCamelCase					(    snake_case_    ,		snake_case_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		str    							=						(
     (
         TFDPRContextEncoder,
         TFDPRQuestionEncoder,
         TFDPRReader,
     )
     if is_tf_available()
     else ()
 )
 __lowercase				:		Tuple    							=						{'''feature-extraction''': TFDPRQuestionEncoder} if is_tf_available() else {}

 __lowercase				:		Dict    							=						False
 __lowercase				:		int    							=						False
 __lowercase				:		Optional[int]    							=						False
 __lowercase				:		Optional[Any]    							=						False
 __lowercase				:		Optional[int]    							=						False


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Dict       =			TFDPRModelTester(self						)
  __A  :		Union[str, Any]       =			ConfigTester(self      ,      config_class=__lowercase      ,      hidden_size=37						)


 def  snake_case__   (      self						):

  """simple docstring"""







  self.config_tester.run_common_tests()


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Union[str, Any]       =			self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_dpr_context_encoder(*__lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Optional[Any]       =			self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_dpr_question_encoder(*__lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Tuple       =			self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_dpr_reader(*__lowercase						)







 @slow
 def  snake_case__   (      self						):

  """simple docstring"""







  for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   __A  :		List[str]       =			TFDPRContextEncoder.from_pretrained(__lowercase						)
   self.assertIsNotNone(__lowercase						)

  for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   __A  :		Any       =			TFDPRContextEncoder.from_pretrained(__lowercase						)
   self.assertIsNotNone(__lowercase						)

  for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   __A  :		Union[str, Any]       =			TFDPRQuestionEncoder.from_pretrained(__lowercase						)
   self.assertIsNotNone(__lowercase						)

  for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   __A  :		Tuple       =			TFDPRReader.from_pretrained(__lowercase						)
   self.assertIsNotNone(__lowercase						)


@require_tf
class 						_lowerCamelCase					(    unittest.TestCase				):




 '''simple docstring'''


 @slow
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Any       =			TFDPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'						)

  __A  :		Dict       =			tf.constant(
      [[101, 7_592, 1_010, 2_003, 2_026, 3_899, 10_140, 1_029, 102]]						)  # [CLS] hello, is my dog cute? [SEP]
  __A  :		Union[str, Any]       =			model(__lowercase						)[0]  # embedding shape = (1, 768)
  # compare the actual values for a slice.
  __A  :		Any       =			tf.constant(
      [
          [
              0.0_3_2_3_6_2_5_3,
              0.1_2_7_5_3_3_3_5,
              0.1_6_8_1_8_5_0_9,
              0.0_0_2_7_9_7_8_6,
              0.3_8_9_6_9_3_3,
              0.2_4_2_6_4_9_4_5,
              0.2_1_7_8_9_7_1,
              -0.0_2_3_3_5_2_2_7,
              -0.0_8_4_8_1_9_5_9,
              -0.1_4_3_2_4_1_1_7,
          ]
      ]						)
  self.assertTrue(numpy.allclose(output[:, :10].numpy()      ,      expected_slice.numpy()      ,      atol=1E-4						)						)










'''simple docstring'''
import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						_lowerCamelCase					(    snake_case_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		Dict    							=						XLMTokenizer
 __lowercase				:		Any    							=						False


 def  snake_case__   (      self						):

  """simple docstring"""







  super().setUp()

  # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  __A  :		Optional[int]       =			[
      'l',
      'o',
      'w',
      'e',
      'r',
      's',
      't',
      'i',
      'd',
      'n',
      'w</w>',
      'r</w>',
      't</w>',
      'lo',
      'low',
      'er</w>',
      'low</w>',
      'lowest</w>',
      'newer</w>',
      'wider</w>',
      '<unk>',
  ]
  __A  :		int       =			dict(zip(__lowercase      ,      range(len(__lowercase						)						)						)						)
  __A  :		str       =			['l o 123', 'lo w 1456', 'e r</w> 1789', '']

  __A  :		Dict       =			os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['vocab_file']						)
  __A  :		List[Any]       =			os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['merges_file']						)
  with open(self.vocab_file      ,      'w'						) as fp:
   fp.write(json.dumps(__lowercase						)						)
  with open(self.merges_file      ,      'w'						) as fp:
   fp.write('\n'.join(__lowercase						)						)


 def  snake_case__   (      self      ,      __lowercase						):

  """simple docstring"""







  __A  :		Any       =			'lower newer'
  __A  :		int       =			'lower newer'
  return input_text, output_text


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Any       =			XLMTokenizer(self.vocab_file      ,      self.merges_file						)

  __A  :		Optional[Any]       =			'lower'
  __A  :		Union[str, Any]       =			['low', 'er</w>']
  __A  :		Any       =			tokenizer.tokenize(__lowercase						)
  self.assertListEqual(__lowercase      ,      __lowercase						)

  __A  :		Optional[int]       =			tokens + ['<unk>']
  __A  :		Any       =			[14, 15, 20]
  self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						)      ,      __lowercase						)







 @slow
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		List[Any]       =			XLMTokenizer.from_pretrained('xlm-mlm-en-2048'						)

  __A  :		Tuple       =			tokenizer.encode('sequence builders'      ,      add_special_tokens=__lowercase						)
  __A  :		Union[str, Any]       =			tokenizer.encode('multi-sequence build'      ,      add_special_tokens=__lowercase						)

  __A  :		int       =			tokenizer.build_inputs_with_special_tokens(__lowercase						)
  __A  :		Tuple       =			tokenizer.build_inputs_with_special_tokens(__lowercase      ,      __lowercase						)

  assert encoded_sentence == [0] + text + [1]
  assert encoded_pair == [0] + text + [1] + text_a + [1]






