
import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


_snake_case      = logging.get_logger(__name__)

_snake_case      = {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_snake_case      = {
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
_snake_case      = {
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
_snake_case      = {
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

_snake_case      = {
    '''facebook/dpr-ctx_encoder-single-nq-base''': 512,
    '''facebook/dpr-ctx_encoder-multiset-base''': 512,
}
_snake_case      = {
    '''facebook/dpr-question_encoder-single-nq-base''': 512,
    '''facebook/dpr-question_encoder-multiset-base''': 512,
}
_snake_case      = {
    '''facebook/dpr-reader-single-nq-base''': 512,
    '''facebook/dpr-reader-multiset-base''': 512,
}


_snake_case      = {
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
_snake_case      = {
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
_snake_case      = {
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}
class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Tuple       						=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_				:					int       						=CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_				:					str       						=CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_				:					List[Any]       						=CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION




class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Union[str, Any]       						=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_				:					List[Any]       						=QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_				:					List[Any]       						=QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_				:					Tuple       						=QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION




_snake_case      = collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

_snake_case      = collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


_snake_case      = r'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''
@add_start_docstrings(__magic_name__							)
class       _lowerCAmelCase  :

 """simple docstring"""

 def __call__(						self			:			Tuple					, SCREAMING_SNAKE_CASE__			:			Optional[int]					, SCREAMING_SNAKE_CASE__			:			Optional[str] = None					, SCREAMING_SNAKE_CASE__			:			Optional[str] = None					, SCREAMING_SNAKE_CASE__			:			Union[bool, str] = False					, SCREAMING_SNAKE_CASE__			:			Union[bool, str] = False					, SCREAMING_SNAKE_CASE__			:			Optional[int] = None					, SCREAMING_SNAKE_CASE__			:			Optional[Union[str, TensorType]] = None					, SCREAMING_SNAKE_CASE__			:			Optional[bool] = None					, **SCREAMING_SNAKE_CASE__			:			Union[str, Any]					, ):






      """simple docstring"""



      if titles is None and texts is None:
           return super().__call__(
               SCREAMING_SNAKE_CASE__					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__					, max_length=SCREAMING_SNAKE_CASE__					, return_tensors=SCREAMING_SNAKE_CASE__					, return_attention_mask=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__					, )
      elif titles is None or texts is None:
           UpperCamelCase						    =					titles if texts is None else texts
           return super().__call__(
               SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__					, max_length=SCREAMING_SNAKE_CASE__					, return_tensors=SCREAMING_SNAKE_CASE__					, return_attention_mask=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__					, )
      UpperCamelCase						    =					titles if not isinstance(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   ) else [titles]
      UpperCamelCase						    =					texts if not isinstance(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   ) else [texts]
      UpperCamelCase						    =					len(SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					questions if not isinstance(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   ) else [questions] * n_passages
      if len(SCREAMING_SNAKE_CASE__   ) != len(SCREAMING_SNAKE_CASE__   ):
           raise ValueError(
               F'There should be as many titles than texts but got {len(SCREAMING_SNAKE_CASE__   )} titles and {len(SCREAMING_SNAKE_CASE__   )} texts.'   )
      UpperCamelCase						    =					super().__call__(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__   )['input_ids']
      UpperCamelCase						    =					super().__call__(SCREAMING_SNAKE_CASE__					, add_special_tokens=SCREAMING_SNAKE_CASE__					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__   )['input_ids']
      UpperCamelCase						    =					{
          'input_ids': [
              (encoded_question_and_title + encoded_text)[:max_length]
              if max_length is not None and truncation
              else encoded_question_and_title + encoded_text
              for encoded_question_and_title, encoded_text in zip(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )
          ]
      }
      if return_attention_mask is not False:
           UpperCamelCase						    =					[]
           for input_ids in encoded_inputs["input_ids"]:
                attention_mask.append([int(input_id != self.pad_token_id   ) for input_id in input_ids]   )
           UpperCamelCase						    =					attention_mask
      return self.pad(SCREAMING_SNAKE_CASE__					, padding=SCREAMING_SNAKE_CASE__					, max_length=SCREAMING_SNAKE_CASE__					, return_tensors=SCREAMING_SNAKE_CASE__   )

 def      __lowerCAmelCase  (						self			:			Dict					, SCREAMING_SNAKE_CASE__			:			BatchEncoding					, SCREAMING_SNAKE_CASE__			:			DPRReaderOutput					, SCREAMING_SNAKE_CASE__			:			int = 16					, SCREAMING_SNAKE_CASE__			:			int = 64					, SCREAMING_SNAKE_CASE__			:			int = 4					, ):






      """simple docstring"""



      UpperCamelCase						    =					reader_input['input_ids']
      UpperCamelCase    ,		UpperCamelCase    ,		UpperCamelCase						    =					reader_output[:3]
      UpperCamelCase						    =					len(SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					sorted(range(SCREAMING_SNAKE_CASE__   )					, reverse=SCREAMING_SNAKE_CASE__					, key=relevance_logits.__getitem__   )
      UpperCamelCase						    =					[]
      for doc_id in sorted_docs:
           UpperCamelCase						    =					list(input_ids[doc_id]   )
           # assuming question & title information is at the beginning of the sequence
           UpperCamelCase						    =					sequence_ids.index(self.sep_token_id					, 2   ) + 1  # second sep id
           if sequence_ids[-1] == self.pad_token_id:
                UpperCamelCase						    =					sequence_ids.index(self.pad_token_id   )
           else:
                UpperCamelCase						    =					len(SCREAMING_SNAKE_CASE__   )

           UpperCamelCase						    =					self._get_best_spans(
               start_logits=start_logits[doc_id][passage_offset:sequence_len]					, end_logits=end_logits[doc_id][passage_offset:sequence_len]					, max_answer_length=SCREAMING_SNAKE_CASE__					, top_spans=SCREAMING_SNAKE_CASE__					, )
           for start_index, end_index in best_spans:
                start_index += passage_offset
                end_index += passage_offset
                nbest_spans_predictions.append(
                    DPRSpanPrediction(
                        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]					, relevance_score=relevance_logits[doc_id]					, doc_id=SCREAMING_SNAKE_CASE__					, start_index=SCREAMING_SNAKE_CASE__					, end_index=SCREAMING_SNAKE_CASE__					, text=self.decode(sequence_ids[start_index : end_index + 1]   )					, )   )
           if len(SCREAMING_SNAKE_CASE__   ) >= num_spans:
                break
      return nbest_spans_predictions[:num_spans]






 def      __lowerCAmelCase  (						self			:			int					, SCREAMING_SNAKE_CASE__			:			List[int]					, SCREAMING_SNAKE_CASE__			:			List[int]					, SCREAMING_SNAKE_CASE__			:			int					, SCREAMING_SNAKE_CASE__			:			int					, ):






      """simple docstring"""



      UpperCamelCase						    =					[]
      for start_index, start_score in enumerate(SCREAMING_SNAKE_CASE__   ):
           for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]   ):
                scores.append(((start_index, start_index + answer_length), start_score + end_score)   )
      UpperCamelCase						    =					sorted(SCREAMING_SNAKE_CASE__					, key=lambda SCREAMING_SNAKE_CASE__   : x[1]					, reverse=SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					[]
      for (start_index, end_index), score in scores:
           if start_index > end_index:
                raise ValueError(F'Wrong span indices: [{start_index}:{end_index}]'   )
           UpperCamelCase						    =					end_index - start_index + 1
           if length > max_answer_length:
                raise ValueError(F'Span is too long: {length} > {max_answer_length}'   )
           if any(
               start_index <= prev_start_index <= prev_end_index <= end_index
               or prev_start_index <= start_index <= end_index <= prev_end_index
               for (prev_start_index, prev_end_index) in chosen_span_intervals   ):
                continue
           chosen_span_intervals.append((start_index, end_index)   )

           if len(SCREAMING_SNAKE_CASE__   ) == top_spans:
                break
      return chosen_span_intervals




@add_end_docstrings(__magic_name__							)
class       _lowerCAmelCase  (		__magic_name__			,	__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Tuple       						=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_				:					Optional[int]       						=READER_PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_				:					Any       						=READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_				:					Optional[Any]       						=READER_PRETRAINED_INIT_CONFIGURATION
 SCREAMING_SNAKE_CASE_				:					str       						=["input_ids", "attention_mask"]




def  __lowerCamelCase					( _lowercase			)       ->      str:
     return "".join(chr(ord(_lowercase			) - 32			) if 'a' <= char <= 'z' else char for char in word			)


if __name__ == "__main__":
 from doctest import testmod

 testmod()


