"""simple docstring"""







import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


SCREAMING_SNAKE_CASE									=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE									=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE									=						{
    'vocab_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
SCREAMING_SNAKE_CASE									=						{
    'vocab_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
SCREAMING_SNAKE_CASE									=						{
    'vocab_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-ctx_encoder-single-nq-base': 512,
    'facebook/dpr-ctx_encoder-multiset-base': 512,
}
SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-question_encoder-single-nq-base': 512,
    'facebook/dpr-question_encoder-multiset-base': 512,
}
SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-reader-single-nq-base': 512,
    'facebook/dpr-reader-multiset-base': 512,
}


SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-ctx_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-ctx_encoder-multiset-base': {'do_lower_case': True},
}
SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-question_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-question_encoder-multiset-base': {'do_lower_case': True},
}
SCREAMING_SNAKE_CASE									=						{
    'facebook/dpr-reader-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-reader-multiset-base': {'do_lower_case': True},
}



class  _lowerCamelCase			(__SCREAMING_SNAKE_CASE     ):
				_snake_case     =  VOCAB_FILES_NAMES
				_snake_case     =  CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
				_snake_case     =  CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case     =  CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
				_snake_case     =  DPRContextEncoderTokenizer



class  _lowerCamelCase			(__SCREAMING_SNAKE_CASE     ):
				_snake_case     =  VOCAB_FILES_NAMES
				_snake_case     =  QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
				_snake_case     =  QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case     =  QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
				_snake_case     =  DPRQuestionEncoderTokenizer



SCREAMING_SNAKE_CASE									=						collections.namedtuple(
    'DPRSpanPrediction', ['span_score', 'relevance_score', 'doc_id', 'start_index', 'end_index', 'text']
)

SCREAMING_SNAKE_CASE									=						collections.namedtuple('DPRReaderOutput', ['start_logits', 'end_logits', 'relevance_logits'])


SCREAMING_SNAKE_CASE									=						r'\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `\'tf\'`: Return TensorFlow `tf.constant` objects.\n                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.\n                - `\'np\'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer\'s default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Return:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    '



@add_start_docstrings(__SCREAMING_SNAKE_CASE     )
class  _lowerCamelCase			:







				def __call__( self    :       Any			,					lowerCamelCase_    :       List[str]			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Union[bool, str] = False			,					lowerCamelCase_    :       Union[bool, str] = False			,					lowerCamelCase_    :       Optional[int] = None			,					lowerCamelCase_    :       Optional[Union[str, TensorType]] = None			,					lowerCamelCase_    :       Optional[bool] = None			,					**lowerCamelCase_    :       Tuple			,					):

					"""simple docstring"""






					if titles is None and texts is None:
						return super().__call__(
						    _a			,					padding=_a			,					truncation=_a			,					max_length=_a			,					return_tensors=_a			,					return_attention_mask=_a			,					**_a			,					)
					elif titles is None or texts is None:
						_lowercase :    int							=							titles if texts is None else texts
						return super().__call__(
						    _a			,					_a			,					padding=_a			,					truncation=_a			,					max_length=_a			,					return_tensors=_a			,					return_attention_mask=_a			,					**_a			,					)
					_lowercase :    List[Any]							=							titles if not isinstance(_a			,					_a						) else [titles]
					_lowercase :    Dict							=							texts if not isinstance(_a			,					_a						) else [texts]
					_lowercase :    Tuple							=							len(_a						)
					_lowercase :    Optional[Any]							=							questions if not isinstance(_a			,					_a						) else [questions] * n_passages
					assert len(_a						) == len(
					    _a						), F'''There should be as many titles than texts but got {len(_a						)} titles and {len(_a						)} texts.'''
					_lowercase :    str							=							super().__call__(_a			,					_a			,					padding=_a			,					truncation=_a						)["""input_ids"""]
					_lowercase :    Union[str, Any]							=							super().__call__(_a			,					add_special_tokens=_a			,					padding=_a			,					truncation=_a						)["""input_ids"""]
					_lowercase :    str							=							{
					    """input_ids""": [
					        (encoded_question_and_title + encoded_text)[:max_length]
					        if max_length is not None and truncation
					        else encoded_question_and_title + encoded_text
					        for encoded_question_and_title, encoded_text in zip(_a			,					_a						)
					    ]
					}
					if return_attention_mask is not False:
						_lowercase :    List[str]							=							[]
						for input_ids in encoded_inputs["input_ids"]:
							attention_mask.append([int(input_id != self.pad_token_id						) for input_id in input_ids]						)
						_lowercase :    int							=							attention_mask
					return self.pad(_a			,					padding=_a			,					max_length=_a			,					return_tensors=_a						)







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       BatchEncoding			,					lowerCamelCase_    :       DPRReaderOutput			,					lowerCamelCase_    :       int = 1_6			,					lowerCamelCase_    :       int = 6_4			,					lowerCamelCase_    :       int = 4			,					):

					"""simple docstring"""






					_lowercase :    Dict							=							reader_input["""input_ids"""]
					_lowercase :    Optional[int]							=							reader_output[:3]
					_lowercase :    Dict							=							len(_a						)
					_lowercase :    Dict							=							sorted(range(_a						)			,					reverse=_a			,					key=relevance_logits.__getitem__						)
					_lowercase :    List[DPRReaderOutput]							=							[]
					for doc_id in sorted_docs:
						_lowercase :    Tuple							=							list(input_ids[doc_id]						)
						# assuming question & title information is at the beginning of the sequence
						_lowercase :    List[Any]							=							sequence_ids.index(self.sep_token_id			,					2						) + 1  # second sep id
						if sequence_ids[-1] == self.pad_token_id:
							_lowercase :    List[str]							=							sequence_ids.index(self.pad_token_id						)
						else:
							_lowercase :    Union[str, Any]							=							len(_a						)

						_lowercase :    Tuple							=							self._get_best_spans(
						    start_logits=start_logits[doc_id][passage_offset:sequence_len]			,					end_logits=end_logits[doc_id][passage_offset:sequence_len]			,					max_answer_length=_a			,					top_spans=_a			,					)
						for start_index, end_index in best_spans:
							start_index += passage_offset
							end_index += passage_offset
							nbest_spans_predictions.append(
							    DPRSpanPrediction(
							        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]			,					relevance_score=relevance_logits[doc_id]			,					doc_id=_a			,					start_index=_a			,					end_index=_a			,					text=self.decode(sequence_ids[start_index : end_index + 1]						)			,					)						)
						if len(_a						) >= num_spans:
							break
					return nbest_spans_predictions[:num_spans]







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       int			,					lowerCamelCase_    :       int			,					):

					"""simple docstring"""






					_lowercase :    List[str]							=							[]
					for start_index, start_score in enumerate(_a						):
						for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]						):
							scores.append(((start_index, start_index + answer_length), start_score + end_score)						)
					_lowercase :    Any							=							sorted(_a			,					key=lambda lowerCamelCase_						: x[1]			,					reverse=_a						)
					_lowercase :    List[str]							=							[]
					for (start_index, end_index), score in scores:
						assert start_index <= end_index, F'''Wrong span indices: [{start_index}:{end_index}]'''
						_lowercase :    List[str]							=							end_index - start_index + 1
						assert length <= max_answer_length, F'''Span is too long: {length} > {max_answer_length}'''
						if any(
						    start_index <= prev_start_index <= prev_end_index <= end_index
						    or prev_start_index <= start_index <= end_index <= prev_end_index
						    for (prev_start_index, prev_end_index) in chosen_span_intervals						):
							continue
						chosen_span_intervals.append((start_index, end_index)						)

						if len(_a						) == top_spans:
							break
					return chosen_span_intervals



@add_end_docstrings(__SCREAMING_SNAKE_CASE     )
class  _lowerCamelCase			(__SCREAMING_SNAKE_CASE						,							__SCREAMING_SNAKE_CASE     ):
				_snake_case     =  VOCAB_FILES_NAMES
				_snake_case     =  READER_PRETRAINED_VOCAB_FILES_MAP
				_snake_case     =  READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case     =  READER_PRETRAINED_INIT_CONFIGURATION
				_snake_case     =  ["input_ids", "attention_mask"]
				_snake_case     =  DPRReaderTokenizer
"""simple docstring"""







import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  ""
				_snake_case     =  (
				    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
				)
				_snake_case     =  None  # compression type in fsspec. ex: "gzip"
				_snake_case     =  None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz







				def __init__( self    :       str			,					lowerCamelCase_    :       str = ""			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Optional[dict] = None			,					**lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					super().__init__(self			,					**lowerCamelCase_						)
					# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
					_lowercase :    Union[str, Any]							=							fsspec.open(
					    lowerCamelCase_			,					mode='rb'			,					protocol=lowerCamelCase_			,					compression=self.compression			,					client_kwargs={
					        'requote_redirect_url': False,  # see https://github.com/huggingface/datasets/pull/5459
					        'trust_env': True,  # Enable reading proxy env variables.
					        **(target_options or {}).pop('client_kwargs'			,					{}						),  # To avoid issues if it was already passed.
					    }			,					**(target_options or {})			,					)
					_lowercase :    str							=							os.path.basename(self.file.path.split('::'						)[0]						)
					_lowercase :    Optional[int]							=							(
					    self.compressed_name[: self.compressed_name.rindex('.'						)]
					    if '.' in self.compressed_name
					    else self.compressed_name
					)
					_lowercase :    str							=							None







				@classmethod
				def 							__UpperCAmelCase					( cls    :       int			,					lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					return super()._strip_protocol(lowerCamelCase_						).lstrip('/'						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					if self.dir_cache is None:
						_lowercase :    Tuple							=							{**self.file.fs.info(self.file.path						), 'name': self.uncompressed_name}
						_lowercase :    int							=							{f['name']: f}







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       str						):

					"""simple docstring"""






					return self.file.open().read()







				def 							__UpperCAmelCase					( self    :       Dict			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       str = "rb"			,					lowerCamelCase_    :       str=None			,					lowerCamelCase_    :       str=True			,					lowerCamelCase_    :       Union[str, Any]=None			,					**lowerCamelCase_    :       Optional[int]			,					):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							self._strip_protocol(lowerCamelCase_						)
					if mode != "rb":
						raise ValueError(F'''Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''''						)
					return self.file.open()



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "bz2"
				_snake_case     =  "bz2"
				_snake_case     =  ".bz2"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "gzip"
				_snake_case     =  "gzip"
				_snake_case     =  ".gz"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "lz4"
				_snake_case     =  "lz4"
				_snake_case     =  ".lz4"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "xz"
				_snake_case     =  "xz"
				_snake_case     =  ".xz"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "zstd"
				_snake_case     =  "zstd"
				_snake_case     =  ".zst"







				def __init__( self    :       Union[str, Any]			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       str = "rb"			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Optional[dict] = None			,					lowerCamelCase_    :       int = DEFAULT_BLOCK_SIZE			,					**lowerCamelCase_    :       int			,					):

					"""simple docstring"""






					super().__init__(
					    fo=lowerCamelCase_			,					mode=lowerCamelCase_			,					target_protocol=lowerCamelCase_			,					target_options=lowerCamelCase_			,					block_size=lowerCamelCase_			,					**lowerCamelCase_			,					)
					# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
					#
					# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
					#     out.close = close
					# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
					#
					# see https://github.com/intake/filesystem_spec/issues/725
					_lowercase :    Any							=							self.file.__enter__



					class  _lowerCamelCase			:







									def __init__( self    :       Any			,					lowerCamelCase_    :       List[Any]						):

										"""simple docstring"""






										_lowercase :    Tuple							=							file_







									def __enter__( self    :       str						):

										"""simple docstring"""






										self._file.__enter__()
										return self







									def __exit__( self    :       Union[str, Any]			,					*lowerCamelCase_    :       Optional[Any]			,					**lowerCamelCase_    :       Union[str, Any]						):

										"""simple docstring"""






										self._file.__exit__(*lowerCamelCase_			,					**lowerCamelCase_						)







									def __iter__( self    :       Optional[int]						):

										"""simple docstring"""






										return iter(self._file						)







									def 							__UpperCAmelCase					( self    :       int						):

										"""simple docstring"""






										return next(self._file						)







									def __getattr__( self    :       List[str]			,					lowerCamelCase_    :       str						):

										"""simple docstring"""






										return getattr(self._file			,					lowerCamelCase_						)

					def fixed_enter(*lowerCamelCase_    :       List[Any]			,					**lowerCamelCase_    :       int						):
						return WrappedFile(_enter(*lowerCamelCase_			,					**lowerCamelCase_						)						)

					_lowercase :    Optional[int]							=							fixed_enter



