'''simple docstring'''






import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


_UpperCAmelCase    :     Tuple         =       logging.get_logger(__name__)


_UpperCAmelCase    :     Optional[int]         =       {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
_UpperCAmelCase    :     List[str]         =       {
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

_UpperCAmelCase    :     int         =       {
    'allenai/led-base-16384': 1_63_84,
}





@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def  _SCREAMING_SNAKE_CASE		(      ):
     _A    		=  (
         list(range(ord('!'				)   ,		ord('~'				) + 1				)				) + list(range(ord('¡'				)   ,		ord('¬'				) + 1				)				) + list(range(ord('®'				)   ,		ord('ÿ'				) + 1				)				)
     )
     _A    		=  bs[:]
     _A    		=  0
     for b in range(2**8				):
          if b not in bs:
               bs.append(A__				)
               cs.append(2**8 + n				)
               n += 1
     _A    		=  [chr(A__				) for n in cs]
     return dict(zip(A__   ,		A__				)				)





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			str				):
     _A    		=  set()
     _A    		=  word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)				)
          _A    		=  char
     return pairs







class lowercase_       (		__snake_case			):







   """simple docstring"""

   __lowerCAmelCase         =   VOCAB_FILES_NAMES
   __lowerCAmelCase         =   PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase         =   ["input_ids", "attention_mask"]
   def __init__(      self      :			Tuple, UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[int], UpperCamelCase__      :			List[Any]="replace", UpperCamelCase__      :			int="<s>", UpperCamelCase__      :			Any="</s>", UpperCamelCase__      :			Optional[Any]="</s>", UpperCamelCase__      :			Optional[int]="<s>", UpperCamelCase__      :			str="<unk>", UpperCamelCase__      :			List[str]="<pad>", UpperCamelCase__      :			List[Any]="<mask>", UpperCamelCase__      :			Tuple=False, **UpperCamelCase__      :			Union[str, Any], )  ->			str:
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else bos_token
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else eos_token
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else sep_token
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else cls_token
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else unk_token
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        _A    		=  AddedToken(__lowerCamelCase, lstrip=__lowerCamelCase, rstrip=__lowerCamelCase     ) if isinstance(__lowerCamelCase, __lowerCamelCase     ) else mask_token

        super().__init__(
            errors=__lowerCamelCase, bos_token=__lowerCamelCase, eos_token=__lowerCamelCase, unk_token=__lowerCamelCase, sep_token=__lowerCamelCase, cls_token=__lowerCamelCase, pad_token=__lowerCamelCase, mask_token=__lowerCamelCase, add_prefix_space=__lowerCamelCase, **__lowerCamelCase, )

        with open(__lowerCamelCase, encoding='utf-8'     ) as vocab_handle:
             _A    		=  json.load(__lowerCamelCase     )
        _A    		=  {v: k for k, v in self.encoder.items()}
        _A    		=  errors  # how to handle errors in decoding
        _A    		=  bytes_to_unicode()
        _A    		=  {v: k for k, v in self.byte_encoder.items()}
        with open(__lowerCamelCase, encoding='utf-8'     ) as merges_handle:
             _A    		=  merges_handle.read().split('\n'     )[1:-1]
        _A    		=  [tuple(merge.split()     ) for merge in bpe_merges]
        _A    		=  dict(zip(__lowerCamelCase, range(len(__lowerCamelCase     )     )     )     )
        _A    		=  {}
        _A    		=  add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        _A    		=  re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )
   @property
   # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
   def 	__UpperCAmelCase   (      self      :			int     )  ->			Optional[Any]:
        return len(self.encoder     )
   def 	__UpperCAmelCase   (      self      :			int     )  ->			Tuple:
        return dict(self.encoder, **self.added_tokens_encoder     )
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			List[str]     )  ->			Union[str, Any]:
        if token in self.cache:
             return self.cache[token]
        _A    		=  tuple(__lowerCamelCase     )
        _A    		=  get_pairs(__lowerCamelCase     )

        if not pairs:
             return token

        while True:
             _A    		=  min(__lowerCamelCase, key=lambda UpperCamelCase__     : self.bpe_ranks.get(__lowerCamelCase, float('inf'     )     )     )
             if bigram not in self.bpe_ranks:
                  break
             _A		,       _A    		=  bigram
             _A    		=  []
             _A    		=  0
             while i < len(__lowerCamelCase     ):
                  try:
                       _A    		=  word.index(__lowerCamelCase, __lowerCamelCase     )
                  except ValueError:
                       new_word.extend(word[i:]     )
                       break
                  else:
                       new_word.extend(word[i:j]     )
                       _A    		=  j

                  if word[i] == first and i < len(__lowerCamelCase     ) - 1 and word[i + 1] == second:
                       new_word.append(first + second     )
                       i += 2
                  else:
                       new_word.append(word[i]     )
                       i += 1
             _A    		=  tuple(__lowerCamelCase     )
             _A    		=  new_word
             if len(__lowerCamelCase     ) == 1:
                  break
             else:
                  _A    		=  get_pairs(__lowerCamelCase     )
        _A    		=  ' '.join(__lowerCamelCase     )
        _A    		=  word
        return word
   def 	__UpperCAmelCase   (      self      :			int, UpperCamelCase__      :			List[Any]     )  ->			Tuple:
        _A    		=  []
        for token in re.findall(self.pat, __lowerCamelCase     ):
             _A    		=  ''.join(
                 self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase     ).split(' '     )     )
        return bpe_tokens
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			Union[str, Any]     )  ->			Optional[int]:
        return self.encoder.get(__lowerCamelCase, self.encoder.get(self.unk_token     )     )
   def 	__UpperCAmelCase   (      self      :			List[Any], UpperCamelCase__      :			int     )  ->			str:
        return self.decoder.get(__lowerCamelCase     )
   def 	__UpperCAmelCase   (      self      :			List[Any], UpperCamelCase__      :			List[str]     )  ->			List[str]:
        _A    		=  ''.join(__lowerCamelCase     )
        _A    		=  bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8', errors=self.errors     )
        return text
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[str] = None     )  ->			List[Any]:
        if not os.path.isdir(__lowerCamelCase     ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
             return
        _A    		=  os.path.join(
            __lowerCamelCase, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
        _A    		=  os.path.join(
            __lowerCamelCase, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

        with open(__lowerCamelCase, 'w', encoding='utf-8'     ) as f:
             f.write(json.dumps(self.encoder, indent=2, sort_keys=__lowerCamelCase, ensure_ascii=__lowerCamelCase     ) + '\n'     )

        _A    		=  0
        with open(__lowerCamelCase, 'w', encoding='utf-8'     ) as writer:
             writer.write('#version: 0.2\n'     )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda UpperCamelCase__     : kv[1]     ):
                  if index != token_index:
                       logger.warning(
                           f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                           ' Please check that the tokenizer is not corrupted!'     )
                       _A    		=  token_index
                  writer.write(' '.join(__lowerCamelCase     ) + '\n'     )
                  index += 1

        return vocab_file, merge_file
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None     )  ->			Optional[int]:
        if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _A    		=  [self.cls_token_id]
        _A    		=  [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None, UpperCamelCase__      :			bool = False     )  ->			Union[str, Any]:
        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=__lowerCamelCase, token_ids_a=__lowerCamelCase, already_has_special_tokens=__lowerCamelCase     )

        if token_ids_a is None:
             return [1] + ([0] * len(__lowerCamelCase     )) + [1]
        return [1] + ([0] * len(__lowerCamelCase     )) + [1, 1] + ([0] * len(__lowerCamelCase     )) + [1]
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None     )  ->			Optional[Any]:
        _A    		=  [self.sep_token_id]
        _A    		=  [self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep     ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]
   def 	__UpperCAmelCase   (      self      :			Any, UpperCamelCase__      :			int, UpperCamelCase__      :			Tuple=False, **UpperCamelCase__      :			List[Any]     )  ->			Dict:
        _A    		=  kwargs.pop('add_prefix_space', self.add_prefix_space     )
        if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase     ) > 0 and not text[0].isspace()):
             _A    		=  ' ' + text
        return (text, kwargs)

   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			Union[Dict[str, EncodedInput], BatchEncoding], UpperCamelCase__      :			Optional[int] = None, UpperCamelCase__      :			PaddingStrategy = PaddingStrategy.DO_NOT_PAD, UpperCamelCase__      :			Optional[int] = None, UpperCamelCase__      :			Optional[bool] = None, )  ->			Dict:
        _A    		=  super()._pad(
            encoded_inputs=__lowerCamelCase, max_length=__lowerCamelCase, padding_strategy=__lowerCamelCase, pad_to_multiple_of=__lowerCamelCase, return_attention_mask=__lowerCamelCase, )

        # Load from model defaults
        if return_attention_mask is None:
             _A    		=  'attention_mask' in self.model_input_names

        if return_attention_mask and "global_attention_mask" in encoded_inputs:
             _A    		=  encoded_inputs[self.model_input_names[0]]
             # `global_attention_mask` need to have the same length as other (sequential) inputs.
             _A    		=  len(encoded_inputs['global_attention_mask']     ) != len(__lowerCamelCase     )

             if needs_to_be_padded:
                  _A    		=  len(__lowerCamelCase     ) - len(encoded_inputs['global_attention_mask']     )

                  if self.padding_side == "right":
                       # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                       _A    		=  (
                           encoded_inputs['global_attention_mask'] + [-1] * difference
                       )
                  elif self.padding_side == "left":
                       _A    		=  [-1] * difference + encoded_inputs[
                           'global_attention_mask'
                       ]
                  else:
                       raise ValueError('Invalid padding strategy:' + str(self.padding_side     )     )

        return encoded_inputs






import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


__A			:					str						=			logging.get_logger(__name__)

__A			:					Optional[Any]						=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A			:					Tuple						=			{
    'vocab_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
__A			:					Optional[Any]						=			{
    'vocab_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
__A			:					str						=			{
    'vocab_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}

__A			:					Optional[Any]						=			{
    'facebook/dpr-ctx_encoder-single-nq-base': 5_1_2,
    'facebook/dpr-ctx_encoder-multiset-base': 5_1_2,
}
__A			:					List[str]						=			{
    'facebook/dpr-question_encoder-single-nq-base': 5_1_2,
    'facebook/dpr-question_encoder-multiset-base': 5_1_2,
}
__A			:					Any						=			{
    'facebook/dpr-reader-single-nq-base': 5_1_2,
    'facebook/dpr-reader-multiset-base': 5_1_2,
}


__A			:					str						=			{
    'facebook/dpr-ctx_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-ctx_encoder-multiset-base': {'do_lower_case': True},
}
__A			:					Any						=			{
    'facebook/dpr-question_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-question_encoder-multiset-base': {'do_lower_case': True},
}
__A			:					Dict						=			{
    'facebook/dpr-reader-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-reader-multiset-base': {'do_lower_case': True},
}





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION




__A			:					Optional[int]						=			collections.namedtuple(
    'DPRSpanPrediction', ['span_score', 'relevance_score', 'doc_id', 'start_index', 'end_index', 'text']
)

__A			:					List[Any]						=			collections.namedtuple('DPRReaderOutput', ['start_logits', 'end_logits', 'relevance_logits'])


__A			:					List[Any]						=			r'\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    ```\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n    ```\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `\'tf\'`: Return TensorFlow `tf.constant` objects.\n                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.\n                - `\'np\'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer\'s default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Returns:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    '





@add_start_docstrings(__snake_case		)
class 			_SCREAMING_SNAKE_CASE :







   '''simple docstring'''




   def __call__(       self					:			int     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Optional[str] = None     , __lowerCamelCase					:			Optional[str] = None     , __lowerCamelCase					:			Union[bool, str] = False     , __lowerCamelCase					:			Union[bool, str] = False     , __lowerCamelCase					:			Optional[int] = None     , __lowerCamelCase					:			Optional[Union[str, TensorType]] = None     , __lowerCamelCase					:			Optional[bool] = None     , **__lowerCamelCase					:			Any     , ):
       if titles is None and texts is None:
           return super().__call__(
               __lowerCamelCase     , padding=__lowerCamelCase     , truncation=__lowerCamelCase     , max_length=__lowerCamelCase     , return_tensors=__lowerCamelCase     , return_attention_mask=__lowerCamelCase     , **__lowerCamelCase     , )
       elif titles is None or texts is None:
           SCREAMING_SNAKE_CASE										=   titles if texts is None else texts
           return super().__call__(
               __lowerCamelCase     , __lowerCamelCase     , padding=__lowerCamelCase     , truncation=__lowerCamelCase     , max_length=__lowerCamelCase     , return_tensors=__lowerCamelCase     , return_attention_mask=__lowerCamelCase     , **__lowerCamelCase     , )
       SCREAMING_SNAKE_CASE										=   titles if not isinstance(__lowerCamelCase     , __lowerCamelCase			) else [titles]
       SCREAMING_SNAKE_CASE										=   texts if not isinstance(__lowerCamelCase     , __lowerCamelCase			) else [texts]
       SCREAMING_SNAKE_CASE										=   len(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   questions if not isinstance(__lowerCamelCase     , __lowerCamelCase			) else [questions] * n_passages
       if len(__lowerCamelCase			) != len(__lowerCamelCase			):
           raise ValueError(
               f"There should be as many titles than texts but got {len(__lowerCamelCase			)} titles and {len(__lowerCamelCase			)} texts."			)
       SCREAMING_SNAKE_CASE										=   super().__call__(__lowerCamelCase     , __lowerCamelCase     , padding=__lowerCamelCase     , truncation=__lowerCamelCase			)["input_ids"]
       SCREAMING_SNAKE_CASE										=   super().__call__(__lowerCamelCase     , add_special_tokens=__lowerCamelCase     , padding=__lowerCamelCase     , truncation=__lowerCamelCase			)["input_ids"]
       SCREAMING_SNAKE_CASE										=   {
           "input_ids": [
               (encoded_question_and_title + encoded_text)[:max_length]
               if max_length is not None and truncation
               else encoded_question_and_title + encoded_text
               for encoded_question_and_title, encoded_text in zip(__lowerCamelCase     , __lowerCamelCase			)
           ]
       }
       if return_attention_mask is not False:
           SCREAMING_SNAKE_CASE										=   []
           for input_ids in encoded_inputs["input_ids"]:
               attention_mask.append([int(input_id != self.pad_token_id			) for input_id in input_ids]			)
           SCREAMING_SNAKE_CASE										=   attention_mask
       return self.pad(__lowerCamelCase     , padding=__lowerCamelCase     , max_length=__lowerCamelCase     , return_tensors=__lowerCamelCase			)





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			BatchEncoding     , __lowerCamelCase					:			DPRReaderOutput     , __lowerCamelCase					:			int = 16     , __lowerCamelCase					:			int = 64     , __lowerCamelCase					:			int = 4     , ):
       SCREAMING_SNAKE_CASE										=   reader_input["input_ids"]
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   reader_output[:3]
       SCREAMING_SNAKE_CASE										=   len(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   sorted(range(__lowerCamelCase			)     , reverse=__lowerCamelCase     , key=relevance_logits.__getitem__			)
       SCREAMING_SNAKE_CASE										=   []
       for doc_id in sorted_docs:
           SCREAMING_SNAKE_CASE										=   list(input_ids[doc_id]			)
           # assuming question & title information is at the beginning of the sequence
           SCREAMING_SNAKE_CASE										=   sequence_ids.index(self.sep_token_id     , 2			) + 1  # second sep id
           if sequence_ids[-1] == self.pad_token_id:
               SCREAMING_SNAKE_CASE										=   sequence_ids.index(self.pad_token_id			)
           else:
               SCREAMING_SNAKE_CASE										=   len(__lowerCamelCase			)

           SCREAMING_SNAKE_CASE										=   self._get_best_spans(
               start_logits=start_logits[doc_id][passage_offset:sequence_len]     , end_logits=end_logits[doc_id][passage_offset:sequence_len]     , max_answer_length=__lowerCamelCase     , top_spans=__lowerCamelCase     , )
           for start_index, end_index in best_spans:
               start_index += passage_offset
               end_index += passage_offset
               nbest_spans_predictions.append(
                   DPRSpanPrediction(
                       span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]     , relevance_score=relevance_logits[doc_id]     , doc_id=__lowerCamelCase     , start_index=__lowerCamelCase     , end_index=__lowerCamelCase     , text=self.decode(sequence_ids[start_index : end_index + 1]			)     , )			)
           if len(__lowerCamelCase			) >= num_spans:
               break
       return nbest_spans_predictions[:num_spans]





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			int     , __lowerCamelCase					:			int     , ):
       SCREAMING_SNAKE_CASE										=   []
       for start_index, start_score in enumerate(__lowerCamelCase			):
           for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]			):
               scores.append(((start_index, start_index + answer_length), start_score + end_score)			)
       SCREAMING_SNAKE_CASE										=   sorted(__lowerCamelCase     , key=lambda __lowerCamelCase			: x[1]     , reverse=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   []
       for (start_index, end_index), score in scores:
           if start_index > end_index:
               raise ValueError(f"Wrong span indices: [{start_index}:{end_index}]"			)
           SCREAMING_SNAKE_CASE										=   end_index - start_index + 1
           if length > max_answer_length:
               raise ValueError(f"Span is too long: {length} > {max_answer_length}"			)
           if any(
               start_index <= prev_start_index <= prev_end_index <= end_index
               or prev_start_index <= start_index <= end_index <= prev_end_index
               for (prev_start_index, prev_end_index) in chosen_span_intervals			):
               continue
           chosen_span_intervals.append((start_index, end_index)			)

           if len(__lowerCamelCase			) == top_spans:
               break
       return chosen_span_intervals





@add_end_docstrings(__snake_case		)
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      READER_PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      READER_PRETRAINED_INIT_CONFIGURATION
   lowerCamelCase__				=      ["input_ids", "attention_mask"]