



'''simple docstring'''




import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


UpperCamelCase__ :			str												=				logging.get_logger(__name__)

UpperCamelCase__ :			Dict												=				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__ :			Optional[Any]												=				{
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
UpperCamelCase__ :			Any												=				{
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
UpperCamelCase__ :			List[Any]												=				{
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase__ :			Union[str, Any]												=				{
    '''facebook/dpr-ctx_encoder-single-nq-base''': 5_12,
    '''facebook/dpr-ctx_encoder-multiset-base''': 5_12,
}
UpperCamelCase__ :			Optional[int]												=				{
    '''facebook/dpr-question_encoder-single-nq-base''': 5_12,
    '''facebook/dpr-question_encoder-multiset-base''': 5_12,
}
UpperCamelCase__ :			Union[str, Any]												=				{
    '''facebook/dpr-reader-single-nq-base''': 5_12,
    '''facebook/dpr-reader-multiset-base''': 5_12,
}


UpperCamelCase__ :			Optional[int]												=				{
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
UpperCamelCase__ :			Optional[int]												=				{
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
UpperCamelCase__ :			str												=				{
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[Any]    					=	VOCAB_FILES_NAMES
			_A   :	List[str]    					=	CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
			_A   :	List[Any]    					=	CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A   :	List[Any]    					=	CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	List[Any]    					=	VOCAB_FILES_NAMES
			_A   :	Union[str, Any]    					=	QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
			_A   :	List[str]    					=	QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A   :	Optional[int]    					=	QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
UpperCamelCase__ :			int												=				collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

UpperCamelCase__ :			Optional[int]												=				collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


UpperCamelCase__ :			Dict												=				R'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''






@add_start_docstrings(lowerCamelCase__					)
class 		_UpperCamelCase						:


			'''simple docstring'''

			def __call__(							self  :				str							,  lowerCAmelCase__  :				Any							,  lowerCAmelCase__  :				Optional[str] = None							,  lowerCAmelCase__  :				Optional[str] = None							,  lowerCAmelCase__  :				Union[bool, str] = False							,  lowerCAmelCase__  :				Union[bool, str] = False							,  lowerCAmelCase__  :				Optional[int] = None							,  lowerCAmelCase__  :				Optional[Union[str, TensorType]] = None							,  lowerCAmelCase__  :				Optional[bool] = None							,  **lowerCAmelCase__  :				int							,  ):





					"""simple docstring"""






					if titles is None and texts is None:
							return super().__call__(
							    lowerCAmelCase__							,  padding=lowerCAmelCase__							,  truncation=lowerCAmelCase__							,  max_length=lowerCAmelCase__							,  return_tensors=lowerCAmelCase__							,  return_attention_mask=lowerCAmelCase__							,  **lowerCAmelCase__							,  )
					elif titles is None or texts is None:
							__SCREAMING_SNAKE_CASE :	Any					 =	titles if texts is None else texts
							return super().__call__(
							    lowerCAmelCase__							,  lowerCAmelCase__							,  padding=lowerCAmelCase__							,  truncation=lowerCAmelCase__							,  max_length=lowerCAmelCase__							,  return_tensors=lowerCAmelCase__							,  return_attention_mask=lowerCAmelCase__							,  **lowerCAmelCase__							,  )
					__SCREAMING_SNAKE_CASE :	Tuple					 =	titles if not isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else [titles]
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	texts if not isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else [texts]
					__SCREAMING_SNAKE_CASE :	List[str]					 =	len(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	str					 =	questions if not isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else [questions] * n_passages
					if len(lowerCAmelCase__		) != len(lowerCAmelCase__		):
							raise ValueError(
							    F"There should be as many titles than texts but got {len(lowerCAmelCase__		)} titles and {len(lowerCAmelCase__		)} texts."		)
					__SCREAMING_SNAKE_CASE :	int					 =	super().__call__(lowerCAmelCase__							,  lowerCAmelCase__							,  padding=lowerCAmelCase__							,  truncation=lowerCAmelCase__		)["""input_ids"""]
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	super().__call__(lowerCAmelCase__							,  add_special_tokens=lowerCAmelCase__							,  padding=lowerCAmelCase__							,  truncation=lowerCAmelCase__		)["""input_ids"""]
					__SCREAMING_SNAKE_CASE :	List[str]					 =	{
					    """input_ids""": [
					        (encoded_question_and_title + encoded_text)[:max_length]
					        if max_length is not None and truncation
					        else encoded_question_and_title + encoded_text
					        for encoded_question_and_title, encoded_text in zip(lowerCAmelCase__							,  lowerCAmelCase__		)
					    ]
					}
					if return_attention_mask is not False:
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	[]
							for input_ids in encoded_inputs["input_ids"]:
									attention_mask.append([int(input_id != self.pad_token_id		) for input_id in input_ids]		)
							__SCREAMING_SNAKE_CASE :	List[Any]					 =	attention_mask
					return self.pad(lowerCAmelCase__							,  padding=lowerCAmelCase__							,  max_length=lowerCAmelCase__							,  return_tensors=lowerCAmelCase__		)

			def UpperCamelCase__      (							self  :				Tuple							,  lowerCAmelCase__  :				BatchEncoding							,  lowerCAmelCase__  :				DPRReaderOutput							,  lowerCAmelCase__  :				int = 1_6							,  lowerCAmelCase__  :				int = 6_4							,  lowerCAmelCase__  :				int = 4							,  ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	List[Any]					 =	reader_input["""input_ids"""]
					__SCREAMING_SNAKE_CASE ,		__SCREAMING_SNAKE_CASE ,		__SCREAMING_SNAKE_CASE :	int					 =	reader_output[:3]
					__SCREAMING_SNAKE_CASE :	str					 =	len(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Optional[int]					 =	sorted(range(lowerCAmelCase__		)							,  reverse=lowerCAmelCase__							,  key=relevance_logits.__getitem__		)
					__SCREAMING_SNAKE_CASE :	List[DPRReaderOutput]					 =	[]
					for doc_id in sorted_docs:
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	list(input_ids[doc_id]		)
							# assuming question & title information is at the beginning of the sequence
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	sequence_ids.index(self.sep_token_id							,  2		) + 1  # second sep id
							if sequence_ids[-1] == self.pad_token_id:
									__SCREAMING_SNAKE_CASE :	Optional[int]					 =	sequence_ids.index(self.pad_token_id		)
							else:
									__SCREAMING_SNAKE_CASE :	Tuple					 =	len(lowerCAmelCase__		)

							__SCREAMING_SNAKE_CASE :	List[Any]					 =	self._get_best_spans(
							    start_logits=start_logits[doc_id][passage_offset:sequence_len]							,  end_logits=end_logits[doc_id][passage_offset:sequence_len]							,  max_answer_length=lowerCAmelCase__							,  top_spans=lowerCAmelCase__							,  )
							for start_index, end_index in best_spans:
									start_index += passage_offset
									end_index += passage_offset
									nbest_spans_predictions.append(
									    DPRSpanPrediction(
									        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]							,  relevance_score=relevance_logits[doc_id]							,  doc_id=lowerCAmelCase__							,  start_index=lowerCAmelCase__							,  end_index=lowerCAmelCase__							,  text=self.decode(sequence_ids[start_index : end_index + 1]		)							,  )		)
							if len(lowerCAmelCase__		) >= num_spans:
									break
					return nbest_spans_predictions[:num_spans]



			def UpperCamelCase__      (							self  :				Union[str, Any]							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				int							,  lowerCAmelCase__  :				int							,  ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	int					 =	[]
					for start_index, start_score in enumerate(lowerCAmelCase__		):
							for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]		):
									scores.append(((start_index, start_index + answer_length), start_score + end_score)		)
					__SCREAMING_SNAKE_CASE :	Dict					 =	sorted(lowerCAmelCase__							,  key=lambda lowerCAmelCase__		: x[1]							,  reverse=lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Dict					 =	[]
					for (start_index, end_index), score in scores:
							if start_index > end_index:
									raise ValueError(F"Wrong span indices: [{start_index}:{end_index}]"		)
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	end_index - start_index + 1
							if length > max_answer_length:
									raise ValueError(F"Span is too long: {length} > {max_answer_length}"		)
							if any(
							    start_index <= prev_start_index <= prev_end_index <= end_index
							    or prev_start_index <= start_index <= end_index <= prev_end_index
							    for (prev_start_index, prev_end_index) in chosen_span_intervals		):
									continue
							chosen_span_intervals.append((start_index, end_index)		)

							if len(lowerCAmelCase__		) == top_spans:
									break
					return chosen_span_intervals






@add_end_docstrings(lowerCamelCase__					)
class 		_UpperCamelCase						(							lowerCamelCase__					,      lowerCamelCase__					):


			'''simple docstring'''

			_A   :	str    					=	VOCAB_FILES_NAMES
			_A   :	str    					=	READER_PRETRAINED_VOCAB_FILES_MAP
			_A   :	Optional[Any]    					=	READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A   :	Tuple    					=	READER_PRETRAINED_INIT_CONFIGURATION
			_A   :	Union[str, Any]    					=	['''input_ids''', '''attention_mask''']




'''simple docstring'''




def        lowerCAmelCase_     (			_lowerCamelCase:     int      ):
		if number > 0:
				raise ValueError("""input must be a negative integer"""      )
		__SCREAMING_SNAKE_CASE :	str					 =	len(bin(_lowerCamelCase      )[3:]      )
		__SCREAMING_SNAKE_CASE :	Any					 =	bin(abs(_lowerCamelCase      ) - (1 << binary_number_length)      )[3:]
		__SCREAMING_SNAKE_CASE :	Optional[int]					 =	(
		    (
		        """1"""
		        + """0""" * (binary_number_length - len(_lowerCamelCase      ))
		        + twos_complement_number
		    )
		    if number < 0
		    else """0"""
		)
		return "0b" + twos_complement_number


if __name__ == "__main__":
						import doctest

						doctest.testmod()