

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=0   ):
      '''simple docstring'''







      if name is None:
            A_							       =		None
      else:
            A_							       =		'''.''' * max(0		,				spaces - 2   ) + '''# {:''' + str(50 - spaces   ) + '''s}'''
            A_							       =		fmt.format(SCREAMING_SNAKE_CASE   )

      # Print and recurse (if needed).
      if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
            if msg is not None:
                  print(SCREAMING_SNAKE_CASE   )
            for k in val.keys():
                  recursive_print(SCREAMING_SNAKE_CASE		,				val[k]		,				spaces + 2   )
      elif isinstance(SCREAMING_SNAKE_CASE		,				torch.Tensor   ):
            print(SCREAMING_SNAKE_CASE		,				''':'''		,				val.size()   )
      else:
            print(SCREAMING_SNAKE_CASE		,				''':'''		,				SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		param.size()
      if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            A_							       =		(num_heads, hidden_size, num_splits) + input_shape[1:]
            A_							       =		param.view(*SCREAMING_SNAKE_CASE   )
            A_							       =		param.transpose(0		,				2   )
            A_							       =		param.transpose(1		,				2   ).contiguous()
      elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            A_							       =		(num_heads, num_splits, hidden_size) + input_shape[1:]
            A_							       =		param.view(*SCREAMING_SNAKE_CASE   )
            A_							       =		param.transpose(0		,				1   ).contiguous()
      A_							       =		param.view(*SCREAMING_SNAKE_CASE   )
      return param







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		{}

      # old versions did not store training args
      A_							       =		input_state_dict.get('''args'''		,				SCREAMING_SNAKE_CASE   )
      if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))

            A_							       =		ds_args.padded_vocab_size
            A_							       =		ds_args.max_position_embeddings
            A_							       =		ds_args.hidden_size
            A_							       =		ds_args.num_layers
            A_							       =		ds_args.num_attention_heads
            A_							       =		ds_args.ffn_hidden_size
            # pprint(config)

      # The number of heads.
      A_							       =		config.n_head
      # The hidden_size per head.
      A_							       =		config.n_embd // config.n_head
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
            A_							       =		input_state_dict['''checkpoint_version''']
      else:
            A_							       =		0.0

      # The model.
      A_							       =		input_state_dict['''model''']
      # The language model.
      A_							       =		model['''language_model''']
      # The embeddings.
      A_							       =		lm['''embedding''']

      # The word embeddings.
      A_							       =		embeddings['''word_embeddings''']['''weight''']
      # Truncate the embedding table to vocab_size rows.
      A_							       =		word_embeddings[: config.vocab_size, :]
      A_							       =		word_embeddings

      # The position embeddings.
      A_							       =		embeddings['''position_embeddings''']['''weight''']
      # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
      A_							       =		pos_embeddings.size(0   )
      if n_positions != config.n_positions:
            raise ValueError(
                f"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"   )
      # Store the position embeddings.
      A_							       =		pos_embeddings

      # The transformer.
      A_							       =		lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

      # The regex to extract layer names.
      A_							       =		re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''   )

      # The simple map of names for "automated" rules.
      A_							       =		{
          '''attention.dense''': '''.attn.c_proj.''',
          '''self_attention.dense''': '''.attn.c_proj.''',
          '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
          '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
      }

      # Extract the layers.
      for key, val in transformer.items():
            # Match the name.
            A_							       =		layer_re.match(SCREAMING_SNAKE_CASE   )

            # Stop if that's not a layer
            if m is None:
                  break

            # The index of the layer.
            A_							       =		int(m.group(1   )   )
            # The name of the operation.
            A_							       =		m.group(2   )
            # Is it a weight or a bias?
            A_							       =		m.group(3   )

            # The name of the layer.
            A_							       =		f"transformer.h.{layer_idx}"

            # For layernorm(s), simply store the layer norm.
            if op_name.endswith('''layernorm'''   ):
                  A_							       =		'''ln_1''' if op_name.startswith('''input'''   ) else '''ln_2'''
                  A_							       =		val

            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                  # Insert a tensor of 1x1xDxD bias.
                  A_							       =		torch.tril(torch.ones((n_positions, n_positions)		,				dtype=torch.floataa   )   ).view(
                      1		,				1		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                  A_							       =		causal_mask

                  # Insert a "dummy" tensor for masked_bias.
                  A_							       =		torch.tensor(-1E4		,				dtype=torch.floataa   )
                  A_							       =		masked_bias

                  A_							       =		fix_query_key_value_ordering(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				3		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                  # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                  A_							       =		out_val.transpose(0		,				1   ).contiguous()
                  # Store.
                  A_							       =		out_val

            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                  A_							       =		fix_query_key_value_ordering(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				3		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                  # Store. No change of shape.
                  A_							       =		out_val

            # Transpose the weights.
            elif weight_or_bias == "weight":
                  A_							       =		megatron_to_transformers[op_name]
                  A_							       =		val.transpose(0		,				1   )

            # Copy the bias.
            elif weight_or_bias == "bias":
                  A_							       =		megatron_to_transformers[op_name]
                  A_							       =		val

    # DEBUG.
      assert config.n_layer == layer_idx + 1

      # The final layernorm.
      A_							       =		transformer['''final_layernorm.weight''']
      A_							       =		transformer['''final_layernorm.bias''']

      # For LM head, transformers' wants the matrix to weight embeddings.
      A_							       =		word_embeddings

      # It should be done!
      return output_state_dict







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		argparse.ArgumentParser()
      parser.add_argument('''--print-checkpoint-structure'''		,				action='''store_true'''   )
      parser.add_argument(
          '''path_to_checkpoint'''		,				type=SCREAMING_SNAKE_CASE		,				help='''Path to the checkpoint file (.zip archive or direct .pt file)'''		,				)
      parser.add_argument(
          '''--config_file'''		,				default=''''''		,				type=SCREAMING_SNAKE_CASE		,				help='''An optional config json file describing the pre-trained model.'''		,				)
      A_							       =		parser.parse_args()

      # Extract the basename.
      A_							       =		os.path.dirname(args.path_to_checkpoint   )

      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(f"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"   )
      if args.path_to_checkpoint.endswith('''.zip'''   ):
            with zipfile.ZipFile(args.path_to_checkpoint		,				'''r'''   ) as checkpoint:
                  with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''   ) as pytorch_dict:
                        A_							       =		torch.load(SCREAMING_SNAKE_CASE		,				map_location='''cpu'''   )
      else:
            A_							       =		torch.load(args.path_to_checkpoint		,				map_location='''cpu'''   )

      A_							       =		input_state_dict.get('''args'''		,				SCREAMING_SNAKE_CASE   )

      # Read the config, or default to the model released by NVIDIA.
      if args.config_file == "":
            if ds_args is not None:
                  if ds_args.bias_gelu_fusion:
                        A_							       =		'''gelu_fast'''
                  elif ds_args.openai_gelu:
                        A_							       =		'''gelu_new'''
                  else:
                        A_							       =		'''gelu'''
            else:
                  # in the very early days this used to be "gelu_new"
                  A_							       =		'''gelu_new'''

            # Spell out all parameters in case the defaults change.
            A_							       =		GPTaConfig(
                vocab_size=50257		,				n_positions=1024		,				n_embd=1024		,				n_layer=24		,				n_head=16		,				n_inner=4096		,				activation_function=SCREAMING_SNAKE_CASE		,				resid_pdrop=0.1		,				embd_pdrop=0.1		,				attn_pdrop=0.1		,				layer_norm_epsilon=1E-5		,				initializer_range=0.02		,				summary_type='''cls_index'''		,				summary_use_proj=SCREAMING_SNAKE_CASE		,				summary_activation=SCREAMING_SNAKE_CASE		,				summary_proj_to_labels=SCREAMING_SNAKE_CASE		,				summary_first_dropout=0.1		,				scale_attn_weights=SCREAMING_SNAKE_CASE		,				use_cache=SCREAMING_SNAKE_CASE		,				bos_token_id=50256		,				eos_token_id=50256		,				)
      else:
            A_							       =		GPTaConfig.from_json_file(args.config_file   )

      A_							       =		['''GPT2LMHeadModel''']

      # Convert.
      print('''Converting'''   )
      A_							       =		convert_megatron_checkpoint(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )

      # Add tokenizer class info to config
      # see https://github.com/huggingface/transformers/issues/13906)
      if ds_args is not None:
            A_							       =		ds_args.tokenizer_type
            if tokenizer_type == "GPT2BPETokenizer":
                  A_							       =		'''gpt2'''
            elif tokenizer_type == "PretrainedFromHF":
                  A_							       =		ds_args.tokenizer_name_or_path
            else:
                  raise ValueError(f"Unrecognized tokenizer_type {tokenizer_type}"   )
      else:
            A_							       =		'''gpt2'''

      A_							       =		AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE   )
      A_							       =		type(SCREAMING_SNAKE_CASE   ).__name__
      A_							       =		tokenizer_class

      # Store the config to file.
      print('''Saving config'''   )
      config.save_pretrained(SCREAMING_SNAKE_CASE   )

      # Save tokenizer based on args
      print(f"Adding {tokenizer_class} tokenizer files"   )
      tokenizer.save_pretrained(SCREAMING_SNAKE_CASE   )

      # Store the state_dict to file.
      A_							       =		os.path.join(SCREAMING_SNAKE_CASE		,				'''pytorch_model.bin'''   )
      print(f"Saving checkpoint to \"{output_checkpoint_file}\""   )
      torch.save(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )


####################################################################################################

if __name__ == "__main__":
      main()

####################################################################################################








import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


__lowercase				      =						logging.get_logger(__name__)

__lowercase				      =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase				      =						{
    """facebook/dpr-ctx_encoder-single-nq-base""": 512,
    """facebook/dpr-ctx_encoder-multiset-base""": 512,
}
__lowercase				      =						{
    """facebook/dpr-question_encoder-single-nq-base""": 512,
    """facebook/dpr-question_encoder-multiset-base""": 512,
}
__lowercase				      =						{
    """facebook/dpr-reader-single-nq-base""": 512,
    """facebook/dpr-reader-multiset-base""": 512,
}


__lowercase				      =						{
    """facebook/dpr-ctx_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-ctx_encoder-multiset-base""": {"""do_lower_case""": True},
}
__lowercase				      =						{
    """facebook/dpr-question_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-question_encoder-multiset-base""": {"""do_lower_case""": True},
}
__lowercase				      =						{
    """facebook/dpr-reader-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-reader-multiset-base""": {"""do_lower_case""": True},
}

class    _lowercase				(					__lowerCamelCase			):
   _lowercase     :       List[Any]  					=    VOCAB_FILES_NAMES
   _lowercase     :       Tuple  					=    CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       int  					=    CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Optional[Any]  					=    CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION



class    _lowercase				(					__lowerCamelCase			):
   _lowercase     :       Union[str, Any]  					=    VOCAB_FILES_NAMES
   _lowercase     :       Any  					=    QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       Optional[int]  					=    QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Dict  					=    QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION



__lowercase				      =						collections.namedtuple(
    """DPRSpanPrediction""", ["""span_score""", """relevance_score""", """doc_id""", """start_index""", """end_index""", """text"""]
)

__lowercase				      =						collections.namedtuple("""DPRReaderOutput""", ["""start_logits""", """end_logits""", """relevance_logits"""])


__lowercase				      =						r"""
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer's default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    """

@add_start_docstrings(__lowerCamelCase			)
class    _lowercase				:

   def __call__(				self	:    str			,						lowerCamelCase__	:    Optional[int]			,						lowerCamelCase__	:    Optional[str] = None			,						lowerCamelCase__	:    Optional[str] = None			,						lowerCamelCase__	:    Union[bool, str] = False			,						lowerCamelCase__	:    Union[bool, str] = False			,						lowerCamelCase__	:    Optional[int] = None			,						lowerCamelCase__	:    Optional[Union[str, TensorType]] = None			,						lowerCamelCase__	:    Optional[bool] = None			,						**lowerCamelCase__	:    Optional[int]			,						) ->			BatchEncoding:
         """simple docstring"""





         if titles is None and texts is None:
               return super().__call__(
                   lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__			,						return_attention_mask=lowerCamelCase__			,						**lowerCamelCase__			,						)
         elif titles is None or texts is None:
               A_							       =		titles if texts is None else texts
               return super().__call__(
                   lowerCamelCase__			,						lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__			,						return_attention_mask=lowerCamelCase__			,						**lowerCamelCase__			,						)
         A_							       =		titles if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [titles]
         A_							       =		texts if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [texts]
         A_							       =		len(lowerCamelCase__       )
         A_							       =		questions if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [questions] * n_passages
         if len(lowerCamelCase__       ) != len(lowerCamelCase__       ):
               raise ValueError(
                   F"There should be as many titles than texts but got {len(lowerCamelCase__       )} titles and {len(lowerCamelCase__       )} texts."       )
         A_							       =		super().__call__(lowerCamelCase__			,						lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__       )['''input_ids''']
         A_							       =		super().__call__(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__       )['''input_ids''']
         A_							       =		{
             '''input_ids''': [
                 (encoded_question_and_title + encoded_text)[:max_length]
                 if max_length is not None and truncation
                 else encoded_question_and_title + encoded_text
                 for encoded_question_and_title, encoded_text in zip(lowerCamelCase__			,						lowerCamelCase__       )
             ]
         }
         if return_attention_mask is not False:
               A_							       =		[]
               for input_ids in encoded_inputs["input_ids"]:
                     attention_mask.append([int(input_id != self.pad_token_id       ) for input_id in input_ids]       )
               A_							       =		attention_mask
         return self.pad(lowerCamelCase__			,						padding=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    int			,						lowerCamelCase__	:    BatchEncoding			,						lowerCamelCase__	:    DPRReaderOutput			,						lowerCamelCase__	:    int = 1_6			,						lowerCamelCase__	:    int = 6_4			,						lowerCamelCase__	:    int = 4			,						) ->			List[DPRSpanPrediction]:
         """simple docstring"""





         A_							       =		reader_input['''input_ids''']
         A_       ,A_       ,A_							       =		reader_output[:3]
         A_							       =		len(lowerCamelCase__       )
         A_							       =		sorted(range(lowerCamelCase__       )			,						reverse=lowerCamelCase__			,						key=relevance_logits.__getitem__       )
         A_							       =		[]
         for doc_id in sorted_docs:
               A_							       =		list(input_ids[doc_id]       )
               # assuming question & title information is at the beginning of the sequence
               A_							       =		sequence_ids.index(self.sep_token_id			,						2       ) + 1  # second sep id
               if sequence_ids[-1] == self.pad_token_id:
                     A_							       =		sequence_ids.index(self.pad_token_id       )
               else:
                     A_							       =		len(lowerCamelCase__       )

               A_							       =		self._get_best_spans(
                   start_logits=start_logits[doc_id][passage_offset:sequence_len]			,						end_logits=end_logits[doc_id][passage_offset:sequence_len]			,						max_answer_length=lowerCamelCase__			,						top_spans=lowerCamelCase__			,						)
               for start_index, end_index in best_spans:
                     start_index += passage_offset
                     end_index += passage_offset
                     nbest_spans_predictions.append(
                         DPRSpanPrediction(
                             span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]			,						relevance_score=relevance_logits[doc_id]			,						doc_id=lowerCamelCase__			,						start_index=lowerCamelCase__			,						end_index=lowerCamelCase__			,						text=self.decode(sequence_ids[start_index : end_index + 1]       )			,						)       )
               if len(lowerCamelCase__       ) >= num_spans:
                     break
         return nbest_spans_predictions[:num_spans]





   def     UpperCamelCase					(				self	:    str			,						lowerCamelCase__	:    List[int]			,						lowerCamelCase__	:    List[int]			,						lowerCamelCase__	:    int			,						lowerCamelCase__	:    int			,						) ->			List[DPRSpanPrediction]:
         """simple docstring"""





         A_							       =		[]
         for start_index, start_score in enumerate(lowerCamelCase__       ):
               for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]       ):
                     scores.append(((start_index, start_index + answer_length), start_score + end_score)       )
         A_							       =		sorted(lowerCamelCase__			,						key=lambda lowerCamelCase__       : x[1]			,						reverse=lowerCamelCase__       )
         A_							       =		[]
         for (start_index, end_index), score in scores:
               if start_index > end_index:
                     raise ValueError(F"Wrong span indices: [{start_index}:{end_index}]"       )
               A_							       =		end_index - start_index + 1
               if length > max_answer_length:
                     raise ValueError(F"Span is too long: {length} > {max_answer_length}"       )
               if any(
                   start_index <= prev_start_index <= prev_end_index <= end_index
                   or prev_start_index <= start_index <= end_index <= prev_end_index
                   for (prev_start_index, prev_end_index) in chosen_span_intervals       ):
                     continue
               chosen_span_intervals.append((start_index, end_index)       )

               if len(lowerCamelCase__       ) == top_spans:
                     break
         return chosen_span_intervals



@add_end_docstrings(__lowerCamelCase			)
class    _lowercase				(					__lowerCamelCase,__lowerCamelCase			):
   _lowercase     :       Any  					=    VOCAB_FILES_NAMES
   _lowercase     :       Any  					=    READER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       Union[str, Any]  					=    READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Union[str, Any]  					=    READER_PRETRAINED_INIT_CONFIGURATION
   _lowercase     :       List[Any]  					=    ['input_ids', 'attention_mask']





