



"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import numpy
     import tensorflow as tf

     from transformers import (
         TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
         TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
         TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
         BertConfig,
         DPRConfig,
         TFDPRContextEncoder,
         TFDPRQuestionEncoder,
         TFDPRReader,
     )



class     UpperCAmelCase    :



   def __init__(       self    :	List[Any]							,       __lowerCamelCase    :	Optional[int]							,       __lowerCamelCase    :	Union[str, Any]=1_3							,       __lowerCamelCase    :	str=7							,       __lowerCamelCase    :	Optional[int]=True							,       __lowerCamelCase    :	str=True							,       __lowerCamelCase    :	int=True							,       __lowerCamelCase    :	List[Any]=True							,       __lowerCamelCase    :	Optional[int]=9_9							,       __lowerCamelCase    :	str=3_2							,       __lowerCamelCase    :	Optional[int]=2							,       __lowerCamelCase    :	Union[str, Any]=4							,       __lowerCamelCase    :	Optional[Any]=3_7							,       __lowerCamelCase    :	List[str]="gelu"							,       __lowerCamelCase    :	List[Any]=0.1							,       __lowerCamelCase    :	int=0.1							,       __lowerCamelCase    :	Any=5_1_2							,       __lowerCamelCase    :	Any=1_6							,       __lowerCamelCase    :	Dict=2							,       __lowerCamelCase    :	Optional[int]=0.0_2							,       __lowerCamelCase    :	Any=3							,       __lowerCamelCase    :	Optional[int]=4							,       __lowerCamelCase    :	Optional[Any]=None							,       __lowerCamelCase    :	List[str]=0							,       ):



     """simple docstring"""
     _snake_case   	=      parent
     _snake_case   	=      batch_size
     _snake_case   	=      seq_length
     _snake_case   	=      is_training
     _snake_case   	=      use_input_mask
     _snake_case   	=      use_token_type_ids
     _snake_case   	=      use_labels
     _snake_case   	=      vocab_size
     _snake_case   	=      hidden_size
     _snake_case   	=      num_hidden_layers
     _snake_case   	=      num_attention_heads
     _snake_case   	=      intermediate_size
     _snake_case   	=      hidden_act
     _snake_case   	=      hidden_dropout_prob
     _snake_case   	=      attention_probs_dropout_prob
     _snake_case   	=      max_position_embeddings
     _snake_case   	=      type_vocab_size
     _snake_case   	=      type_sequence_label_size
     _snake_case   	=      initializer_range
     _snake_case   	=      num_labels
     _snake_case   	=      num_choices
     _snake_case   	=      scope
     _snake_case   	=      projection_dim



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      ids_tensor([self.batch_size, self.seq_length]							,       self.vocab_size				)

     _snake_case   	=      None
     if self.use_input_mask:
       # follow test_modeling_tf_ctrl.py
       _snake_case   	=      random_attention_mask([self.batch_size, self.seq_length]				)

     _snake_case   	=      None
     if self.use_token_type_ids:
       _snake_case   	=      ids_tensor([self.batch_size, self.seq_length]							,       self.type_vocab_size				)

     _snake_case   	=      None
     _snake_case   	=      None
     _snake_case   	=      None
     if self.use_labels:
       _snake_case   	=      ids_tensor([self.batch_size]							,       self.type_sequence_label_size				)
       _snake_case   	=      ids_tensor([self.batch_size, self.seq_length]							,       self.num_labels				)
       _snake_case   	=      ids_tensor([self.batch_size]							,       self.num_choices				)

     _snake_case   	=      BertConfig(
         vocab_size=self.vocab_size							,       hidden_size=self.hidden_size							,       num_hidden_layers=self.num_hidden_layers							,       num_attention_heads=self.num_attention_heads							,       intermediate_size=self.intermediate_size							,       hidden_act=self.hidden_act							,       hidden_dropout_prob=self.hidden_dropout_prob							,       attention_probs_dropout_prob=self.attention_probs_dropout_prob							,       max_position_embeddings=self.max_position_embeddings							,       type_vocab_size=self.type_vocab_size							,       is_decoder=__lowerCamelCase							,       initializer_range=self.initializer_range							,       )
     _snake_case   	=      DPRConfig(projection_dim=self.projection_dim							,       **config.to_dict()				)

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	Union[str, Any]							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      TFDPRContextEncoder(config=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       token_type_ids=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase							,       token_type_ids=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase				)
     self.parent.assertEqual(result.pooler_output.shape							,       (self.batch_size, self.projection_dim or self.hidden_size)				)



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      TFDPRQuestionEncoder(config=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       token_type_ids=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase							,       token_type_ids=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase				)
     self.parent.assertEqual(result.pooler_output.shape							,       (self.batch_size, self.projection_dim or self.hidden_size)				)



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      TFDPRReader(config=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase				)

     self.parent.assertEqual(result.start_logits.shape							,       (self.batch_size, self.seq_length)				)
     self.parent.assertEqual(result.end_logits.shape							,       (self.batch_size, self.seq_length)				)
     self.parent.assertEqual(result.relevance_logits.shape							,       (self.batch_size,)				)





   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      self.prepare_config_and_inputs()
     (
         (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     
     )   	=      config_and_inputs
     _snake_case   	=      {'''input_ids''': input_ids}
     return config, inputs_dict



@require_tf
class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	Any           =  (
       (
           TFDPRContextEncoder,
           TFDPRQuestionEncoder,
           TFDPRReader,
       )
       if is_tf_available()
       else ()
   )
   A__		:	Optional[int]           =  {'''feature-extraction''': TFDPRQuestionEncoder} if is_tf_available() else {}

   A__		:	List[Any]           =  False
   A__		:	Any           =  False
   A__		:	Optional[int]           =  False
   A__		:	Optional[Any]           =  False
   A__		:	str           =  False



   def    __UpperCAmelCase    (       self    :	Dict				):



     """simple docstring"""
     _snake_case   	=      TFDPRModelTester(self				)
     _snake_case   	=      ConfigTester(self							,       config_class=__lowerCamelCase							,       hidden_size=3_7				)



   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     self.config_tester.run_common_tests()



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_dpr_context_encoder(*__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_dpr_question_encoder(*__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     _snake_case   	=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_dpr_reader(*__lowerCamelCase				)





   @slow
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _snake_case   	=      TFDPRContextEncoder.from_pretrained(__lowerCamelCase				)
       self.assertIsNotNone(__lowerCamelCase				)

     for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _snake_case   	=      TFDPRContextEncoder.from_pretrained(__lowerCamelCase				)
       self.assertIsNotNone(__lowerCamelCase				)

     for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _snake_case   	=      TFDPRQuestionEncoder.from_pretrained(__lowerCamelCase				)
       self.assertIsNotNone(__lowerCamelCase				)

     for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _snake_case   	=      TFDPRReader.from_pretrained(__lowerCamelCase				)
       self.assertIsNotNone(__lowerCamelCase				)



@require_tf
class     UpperCAmelCase    (	unittest.TestCase  ):



   @slow
   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     _snake_case   	=      TFDPRQuestionEncoder.from_pretrained('''facebook/dpr-question_encoder-single-nq-base'''				)

     _snake_case   	=      tf.constant(
         [[1_0_1, 7_5_9_2, 1_0_1_0, 2_0_0_3, 2_0_2_6, 3_8_9_9, 1_0_1_4_0, 1_0_2_9, 1_0_2]]				)  # [CLS] hello, is my dog cute? [SEP]
     _snake_case   	=      model(__lowerCamelCase				)[0]  # embedding shape = (1, 768)
     # compare the actual values for a slice.
     _snake_case   	=      tf.constant(
         [
             [
                 0.0_3_2_3_6_2_5_3,
                 0.1_2_7_5_3_3_3_5,
                 0.1_6_8_1_8_5_0_9,
                 0.0_0_2_7_9_7_8_6,
                 0.3_8_9_6_9_3_3,
                 0.2_4_2_6_4_9_4_5,
                 0.2_1_7_8_9_7_1,
                 -0.0_2_3_3_5_2_2_7,
                 -0.0_8_4_8_1_9_5_9,
                 -0.1_4_3_2_4_1_1_7,
             ]
         ]				)
     self.assertTrue(numpy.allclose(output[:, :1_0].numpy()							,       expected_slice.numpy()							,       atol=1E-4				)				)










import argparse
from collections import defaultdict

import yaml


SCREAMING_SNAKE_CASE__   			=   '''docs/source/en/_toctree.yml'''


def A    (							__UpperCamelCase       )    ->   Optional[Any]:
			A__	  =     defaultdict(__UpperCamelCase       )
			for doc in model_doc:
						counts[doc["local"]] += 1
			A__	  =     [key for key, value in counts.items() if value > 1]

			A__	  =     []
			for duplicate_key in duplicates:
						A__	  =     list({doc['title'] for doc in model_doc if doc['local'] == duplicate_key}       )
						if len(__UpperCamelCase       ) > 1:
									raise ValueError(
									    f'''{duplicate_key} is present several times in the documentation table of content at '''
									    '`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '
									    'others.'       )
						# Only add this once
						new_doc.append({'local': duplicate_key, 'title': titles[0]}       )

			# Add none duplicate-keys
			new_doc.extend([doc for doc in model_doc if counts[doc['local']] == 1]       )

			# Sort
			return sorted(__UpperCamelCase			,						key=lambda __UpperCamelCase       : s["title"].lower()       )


def A    (							__UpperCamelCase=False       )    ->   str:
			with open(__UpperCamelCase			,						encoding='utf-8'       ) as f:
						A__	  =     yaml.safe_load(f.read()       )

			# Get to the API doc
			A__	  =     0
			while content[api_idx]["title"] != "API":
						api_idx += 1
			A__	  =     content[api_idx]['sections']

			# Then to the model doc
			A__	  =     0
			while api_doc[model_idx]["title"] != "Models":
						model_idx += 1

			A__	  =     api_doc[model_idx]['sections']

			A__	  =     [(idx, section) for idx, section in enumerate(__UpperCamelCase       ) if 'sections' in section]
			A__	  =     False
			for idx, modality_doc in modalities_docs:
						A__	  =     modality_doc['sections']
						A__	  =     clean_model_doc_toc(__UpperCamelCase       )

						if old_modality_doc != new_modality_doc:
									A__	  =     True
									if overwrite:
												A__	  =     new_modality_doc

			if diff:
						if overwrite:
									A__	  =     model_doc
									A__	  =     api_doc
									with open(__UpperCamelCase			,						'w'			,						encoding='utf-8'       ) as f:
												f.write(yaml.dump(__UpperCamelCase			,						allow_unicode=__UpperCamelCase       )       )
						else:
									raise ValueError(
									    'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'       )


if __name__ == "__main__":
							SCREAMING_SNAKE_CASE__   			=   argparse.ArgumentParser()
							parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
							SCREAMING_SNAKE_CASE__   			=   parser.parse_args()

							check_model_doc(args.fix_and_overwrite)




