






import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule


class 							__UpperCamelCase     (  _lowerCAmelCase ):
	__snake_case						:BigBirdConfig
	__snake_case						:jnp.dtype =							jnp.floataa
	__snake_case						:bool =							True







	def 			_a (	self   :				Optional[Any]      )		->			Optional[Any]:
		"""simple docstring"""







		super().setup()
		__lowercase   							=	nn.Dense(5							,					dtype=self.dtype      )







	def __call__(	self   :				Optional[int]							,					*_lowerCAmelCase   :				List[str]							,					**_lowerCAmelCase   :				Optional[Any]      )		->			Dict:
		"""simple docstring"""







		__lowercase   							=	super().__call__(*_lowerCAmelCase							,					**_lowerCAmelCase      )
		__lowercase   							=	self.cls(outputs[2]      )
		return outputs[:2] + (cls_out,)


class 							__UpperCamelCase     (  _lowerCAmelCase ):
	__snake_case						:Tuple =							FlaxBigBirdForNaturalQuestionsModule





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	def cross_entropy(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase=None       ):
		__lowercase   							=	logits.shape[-1]
		__lowercase   							=	(labels[..., None] == jnp.arange(lowerCamelCase       )[None]).astype("""f4"""       )
		__lowercase   							=	jax.nn.log_softmax(lowerCamelCase					,	axis=-1       )
		__lowercase   							=	-jnp.sum(labels * logits					,	axis=-1       )
		if reduction is not None:
			__lowercase   							=	reduction(lowerCamelCase       )
		return loss

	__lowercase   							=	partial(lowerCamelCase					,	reduction=jnp.mean       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	return (start_loss + end_loss + pooled_loss) / 3


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:str =							"google/bigbird-roberta-base"
	__snake_case						:int =							3_0_0_0
	__snake_case						:int =							1_0_5_0_0

	__snake_case						:int =							1_2_8
	__snake_case						:int =							3

	__snake_case						:int =							1
	__snake_case						:int =							5

	# tx_args
	__snake_case						:float =							3e-5
	__snake_case						:float =							0.0
	__snake_case						:int =							2_0_0_0_0
	__snake_case						:float =							0.00_95

	__snake_case						:str =							"bigbird-roberta-natural-questions"
	__snake_case						:str =							"training-expt"
	__snake_case						:str =							"data/nq-training.jsonl"
	__snake_case						:str =							"data/nq-validation.jsonl"







	def 			_a (	self   :				Dict      )		->			List[str]:
		"""simple docstring"""







		os.makedirs(self.base_dir							,					exist_ok=_lowerCAmelCase      )
		__lowercase   							=	os.path.join(self.base_dir							,					self.save_dir      )
		__lowercase   							=	self.batch_size_per_device * jax.device_count()


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:int
	__snake_case						:int =							4_0_9_6  # no dynamic padding on TPUs







	def __call__(	self   :				Any							,					_lowerCAmelCase   :				str      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	self.collate_fn(_lowerCAmelCase      )
		__lowercase   							=	jax.tree_util.tree_map(_lowerCAmelCase							,					_lowerCAmelCase      )
		return batch







	def 			_a (	self   :				Dict							,					_lowerCAmelCase   :				Tuple      )		->			Optional[int]:
		"""simple docstring"""







		__lowercase							,    __lowercase   							=	self.fetch_inputs(features["""input_ids"""]      )
		__lowercase   							=	{
		    """input_ids""": jnp.array(_lowerCAmelCase							,					dtype=jnp.intaa      ),
		    """attention_mask""": jnp.array(_lowerCAmelCase							,					dtype=jnp.intaa      ),
		    """start_labels""": jnp.array(features["""start_token"""]							,					dtype=jnp.intaa      ),
		    """end_labels""": jnp.array(features["""end_token"""]							,					dtype=jnp.intaa      ),
		    """pooled_labels""": jnp.array(features["""category"""]							,					dtype=jnp.intaa      ),
		}
		return batch







	def 			_a (	self   :				Optional[Any]							,					_lowerCAmelCase   :				list      )		->			Optional[Any]:
		"""simple docstring"""







		__lowercase   							=	[self._fetch_inputs(_lowerCAmelCase      ) for ids in input_ids]
		return zip(*_lowerCAmelCase      )







	def 			_a (	self   :				int							,					_lowerCAmelCase   :				list      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	[1 for _ in range(len(_lowerCAmelCase      )      )]
		while len(_lowerCAmelCase      ) < self.max_length:
			input_ids.append(self.pad_id      )
			attention_mask.append(0      )
		return input_ids, attention_mask





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase=None       ):




	'''simple docstring'''







	if seed is not None:
		__lowercase   							=	dataset.shuffle(seed=lowerCamelCase       )
	for i in range(len(lowerCamelCase       ) // batch_size       ):
		__lowercase   							=	dataset[i * batch_size : (i + 1) * batch_size]
		yield dict(lowerCamelCase       )





@partial(jax.pmap					,	axis_name="""batch"""       )
def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	**lowerCamelCase       ):




	'''simple docstring'''







	def loss_fn(lowerCamelCase       ):
		__lowercase   							=	model_inputs.pop("""start_labels"""       )
		__lowercase   							=	model_inputs.pop("""end_labels"""       )
		__lowercase   							=	model_inputs.pop("""pooled_labels"""       )

		__lowercase   							=	state.apply_fn(**lowerCamelCase					,	params=lowerCamelCase					,	dropout_rng=lowerCamelCase					,	train=lowerCamelCase       )
		__lowercase							,    __lowercase							,    __lowercase   							=	outputs

		return state.loss_fn(
		    lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	)

	__lowercase							,    __lowercase   							=	jax.random.split(lowerCamelCase       )
	__lowercase   							=	jax.value_and_grad(lowerCamelCase       )
	__lowercase							,    __lowercase   							=	grad_fn(state.params       )
	__lowercase   							=	jax.lax.pmean({"""loss""": loss}					,	axis_name="""batch"""       )
	__lowercase   							=	jax.lax.pmean(lowerCamelCase					,	"""batch"""       )

	__lowercase   							=	state.apply_gradients(grads=lowerCamelCase       )
	return state, metrics, new_drp_rng





@partial(jax.pmap					,	axis_name="""batch"""       )
def 					snake_case	(  lowerCamelCase					,	**lowerCamelCase       ):




	'''simple docstring'''







	__lowercase   							=	model_inputs.pop("""start_labels"""       )
	__lowercase   							=	model_inputs.pop("""end_labels"""       )
	__lowercase   							=	model_inputs.pop("""pooled_labels"""       )

	__lowercase   							=	state.apply_fn(**lowerCamelCase					,	params=state.params					,	train=lowerCamelCase       )
	__lowercase							,    __lowercase							,    __lowercase   							=	outputs

	__lowercase   							=	state.loss_fn(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	jax.lax.pmean({"""loss""": loss}					,	axis_name="""batch"""       )
	return metrics


class 							__UpperCamelCase     (  train_state.TrainState ):
	__snake_case						:Callable =							struct.field(pytree_node=_lowerCAmelCase )


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:Args
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:wandb
	__snake_case						:Callable =							None







	def 			_a (	self   :				Optional[Any]							,					_lowerCAmelCase   :				List[str]							,					_lowerCAmelCase   :				Optional[int]							,					_lowerCAmelCase   :				Tuple							,					_lowerCAmelCase   :				Tuple=None      )		->			int:
		"""simple docstring"""







		__lowercase   							=	model.params
		__lowercase   							=	TrainState.create(
		    apply_fn=model.__call__							,					params=_lowerCAmelCase							,					tx=_lowerCAmelCase							,					loss_fn=_lowerCAmelCase							,					)
		if ckpt_dir is not None:
			__lowercase							,    __lowercase							,    __lowercase							,    __lowercase							,    __lowercase   							=	restore_checkpoint(_lowerCAmelCase							,					_lowerCAmelCase      )
			__lowercase   							=	{
			    """lr""": args.lr,
			    """init_lr""": args.init_lr,
			    """warmup_steps""": args.warmup_steps,
			    """num_train_steps""": num_train_steps,
			    """weight_decay""": args.weight_decay,
			}
			__lowercase							,    __lowercase   							=	build_tx(**_lowerCAmelCase      )
			__lowercase   							=	train_state.TrainState(
			    step=_lowerCAmelCase							,					apply_fn=model.__call__							,					params=_lowerCAmelCase							,					tx=_lowerCAmelCase							,					opt_state=_lowerCAmelCase							,					)
			__lowercase   							=	args
			__lowercase   							=	data_collator
			__lowercase   							=	lr
			__lowercase   							=	params
		__lowercase   							=	jax_utils.replicate(_lowerCAmelCase      )
		return state







	def 			_a (	self   :				str							,					_lowerCAmelCase   :				Any							,					_lowerCAmelCase   :				str							,					_lowerCAmelCase   :				List[str]      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	self.args
		__lowercase   							=	len(_lowerCAmelCase      ) // args.batch_size

		__lowercase   							=	jax.random.PRNGKey(0      )
		__lowercase   							=	jax.random.split(_lowerCAmelCase							,					jax.device_count()      )
		for epoch in range(args.max_epochs      ):
			__lowercase   							=	jnp.array(0							,					dtype=jnp.floataa      )
			__lowercase   							=	get_batched_dataset(_lowerCAmelCase							,					args.batch_size							,					seed=_lowerCAmelCase      )
			__lowercase   							=	0
			for batch in tqdm(_lowerCAmelCase							,					total=_lowerCAmelCase							,					desc=F'Running EPOCH-{epoch}'      ):
				__lowercase   							=	self.data_collator(_lowerCAmelCase      )
				__lowercase							,    __lowercase							,    __lowercase   							=	self.train_step_fn(_lowerCAmelCase							,					_lowerCAmelCase							,					**_lowerCAmelCase      )
				running_loss += jax_utils.unreplicate(metrics["""loss"""]      )
				i += 1
				if i % args.logging_steps == 0:
					__lowercase   							=	jax_utils.unreplicate(state.step      )
					__lowercase   							=	running_loss.item() / i
					__lowercase   							=	self.scheduler_fn(state_step - 1      )

					__lowercase   							=	self.evaluate(_lowerCAmelCase							,					_lowerCAmelCase      )
					__lowercase   							=	{
					    """step""": state_step.item(),
					    """eval_loss""": eval_loss.item(),
					    """tr_loss""": tr_loss,
					    """lr""": lr.item(),
					}
					tqdm.write(str(_lowerCAmelCase      )      )
					self.logger.log(_lowerCAmelCase							,					commit=_lowerCAmelCase      )

				if i % args.save_steps == 0:
					self.save_checkpoint(args.save_dir + F'-e{epoch}-s{i}'							,					state=_lowerCAmelCase      )







	def 			_a (	self   :				int							,					_lowerCAmelCase   :				Union[str, Any]							,					_lowerCAmelCase   :				Dict      )		->			List[Any]:
		"""simple docstring"""







		__lowercase   							=	get_batched_dataset(_lowerCAmelCase							,					self.args.batch_size      )
		__lowercase   							=	len(_lowerCAmelCase      ) // self.args.batch_size
		__lowercase   							=	jnp.array(0							,					dtype=jnp.floataa      )
		__lowercase   							=	0
		for batch in tqdm(_lowerCAmelCase							,					total=_lowerCAmelCase							,					desc="""Evaluating ... """      ):
			__lowercase   							=	self.data_collator(_lowerCAmelCase      )
			__lowercase   							=	self.val_step_fn(_lowerCAmelCase							,					**_lowerCAmelCase      )
			running_loss += jax_utils.unreplicate(metrics["""loss"""]      )
			i += 1
		return running_loss / i







	def 			_a (	self   :				Union[str, Any]							,					_lowerCAmelCase   :				Union[str, Any]							,					_lowerCAmelCase   :				List[str]      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	jax_utils.unreplicate(_lowerCAmelCase      )
		print(F'SAVING CHECKPOINT IN {save_dir}'							,					end=""" ... """      )
		self.model_save_fn(_lowerCAmelCase							,					params=state.params      )
		with open(os.path.join(_lowerCAmelCase							,					"""opt_state.msgpack"""      )							,					"""wb"""      ) as f:
			f.write(to_bytes(state.opt_state      )      )
		joblib.dump(self.args							,					os.path.join(_lowerCAmelCase							,					"""args.joblib"""      )      )
		joblib.dump(self.data_collator							,					os.path.join(_lowerCAmelCase							,					"""data_collator.joblib"""      )      )
		with open(os.path.join(_lowerCAmelCase							,					"""training_state.json"""      )							,					"""w"""      ) as f:
			json.dump({"""step""": state.step.item()}							,					_lowerCAmelCase      )
		print("""DONE"""      )





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	print(F'RESTORING CHECKPOINT FROM {save_dir}'					,	end=""" ... """       )
	with open(os.path.join(lowerCamelCase					,	"""flax_model.msgpack"""       )					,	"""rb"""       ) as f:
		__lowercase   							=	from_bytes(state.params					,	f.read()       )

	with open(os.path.join(lowerCamelCase					,	"""opt_state.msgpack"""       )					,	"""rb"""       ) as f:
		__lowercase   							=	from_bytes(state.opt_state					,	f.read()       )

	__lowercase   							=	joblib.load(os.path.join(lowerCamelCase					,	"""args.joblib"""       )       )
	__lowercase   							=	joblib.load(os.path.join(lowerCamelCase					,	"""data_collator.joblib"""       )       )

	with open(os.path.join(lowerCamelCase					,	"""training_state.json"""       )					,	"""r"""       ) as f:
		__lowercase   							=	json.load(lowerCamelCase       )
	__lowercase   							=	training_state["""step"""]

	print("""DONE"""       )
	return params, opt_state, step, args, data_collator





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	__lowercase   							=	num_train_steps - warmup_steps
	__lowercase   							=	optax.linear_schedule(init_value=lowerCamelCase					,	end_value=lowerCamelCase					,	transition_steps=lowerCamelCase       )
	__lowercase   							=	optax.linear_schedule(init_value=lowerCamelCase					,	end_value=1e-7					,	transition_steps=lowerCamelCase       )
	__lowercase   							=	optax.join_schedules(schedules=[warmup_fn, decay_fn]					,	boundaries=[warmup_steps]       )
	return lr





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	def weight_decay_mask(lowerCamelCase       ):
		__lowercase   							=	traverse_util.flatten_dict(lowerCamelCase       )
		__lowercase   							=	{k: (v[-1] != """bias""" and v[-2:] != ("""LayerNorm""", """scale""")) for k, v in params.items()}
		return traverse_util.unflatten_dict(lowerCamelCase       )

	__lowercase   							=	scheduler_fn(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       )

	__lowercase   							=	optax.adamw(learning_rate=lowerCamelCase					,	weight_decay=lowerCamelCase					,	mask=lowerCamelCase       )
	return tx, lr









import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


lowerCamelCase      :       int    =						logging.get_logger(__name__)

lowerCamelCase      :       Optional[int]    =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase      :       List[Any]    =						{
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase      :       int    =						{
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase      :       Optional[int]    =						{
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase      :       str    =						{
    '''facebook/dpr-ctx_encoder-single-nq-base''': 512,
    '''facebook/dpr-ctx_encoder-multiset-base''': 512,
}
lowerCamelCase      :       List[str]    =						{
    '''facebook/dpr-question_encoder-single-nq-base''': 512,
    '''facebook/dpr-question_encoder-multiset-base''': 512,
}
lowerCamelCase      :       Union[str, Any]    =						{
    '''facebook/dpr-reader-single-nq-base''': 512,
    '''facebook/dpr-reader-multiset-base''': 512,
}


lowerCamelCase      :       Dict    =						{
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCamelCase      :       Union[str, Any]    =						{
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCamelCase      :       Optional[Any]    =						{
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				DPRContextEncoderTokenizer



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				DPRQuestionEncoderTokenizer

lowerCamelCase      :       Dict    =						collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

lowerCamelCase      :       int    =						collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


lowerCamelCase      :       Optional[int]    =						R'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Return:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''



@add_start_docstrings(a_      )
class 				_UpperCamelCase      :


      def __call__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    __UpperCamelCase = False       ,    __UpperCamelCase = False       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    __UpperCamelCase = None       ,    **__UpperCamelCase       ,    )->						BatchEncoding:
           if titles is None and texts is None:
                return super().__call__(
                    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase       ,    return_attention_mask=__UpperCamelCase       ,    **__UpperCamelCase       ,    )
           elif titles is None or texts is None:
                __lowerCAmelCase             =       titles if texts is None else texts
                return super().__call__(
                    __UpperCamelCase       ,    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase       ,    return_attention_mask=__UpperCamelCase       ,    **__UpperCamelCase       ,    )
           __lowerCAmelCase             =       titles if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [titles]
           __lowerCAmelCase             =       texts if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [texts]
           __lowerCAmelCase             =       len(__UpperCamelCase						)
           __lowerCAmelCase             =       questions if not isinstance(__UpperCamelCase       ,    __UpperCamelCase						) else [questions] * n_passages
           assert len(__UpperCamelCase						) == len(
               __UpperCamelCase						), F"""There should be as many titles than texts but got {len(__UpperCamelCase						)} titles and {len(__UpperCamelCase						)} texts."""
           __lowerCAmelCase             =       super().__call__(__UpperCamelCase       ,    __UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase						)["input_ids"]
           __lowerCAmelCase             =       super().__call__(__UpperCamelCase       ,    add_special_tokens=__UpperCamelCase       ,    padding=__UpperCamelCase       ,    truncation=__UpperCamelCase						)["input_ids"]
           __lowerCAmelCase             =       {
               "input_ids": [
                   (encoded_question_and_title + encoded_text)[:max_length]
                   if max_length is not None and truncation
                   else encoded_question_and_title + encoded_text
                   for encoded_question_and_title, encoded_text in zip(__UpperCamelCase       ,    __UpperCamelCase						)
               ]
           }
           if return_attention_mask is not False:
                __lowerCAmelCase             =       []
                for input_ids in encoded_inputs["input_ids"]:
                     attention_mask.append([int(input_id != self.pad_token_id						) for input_id in input_ids]						)
                __lowerCAmelCase             =       attention_mask
           return self.pad(__UpperCamelCase       ,    padding=__UpperCamelCase       ,    max_length=__UpperCamelCase       ,    return_tensors=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase = 1_6       ,    __UpperCamelCase = 6_4       ,    __UpperCamelCase = 4       ,    )->						List[DPRSpanPrediction]:
           __lowerCAmelCase             =       reader_input["input_ids"]
           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       reader_output[:3]
           __lowerCAmelCase             =       len(__UpperCamelCase						)
           __lowerCAmelCase             =       sorted(range(__UpperCamelCase						)       ,    reverse=__UpperCamelCase       ,    key=relevance_logits.__getitem__						)
           __lowerCAmelCase             =       []
           for doc_id in sorted_docs:
                __lowerCAmelCase             =       list(input_ids[doc_id]						)
                # assuming question & title information is at the beginning of the sequence
                __lowerCAmelCase             =       sequence_ids.index(self.sep_token_id       ,    2						) + 1  # second sep id
                if sequence_ids[-1] == self.pad_token_id:
                     __lowerCAmelCase             =       sequence_ids.index(self.pad_token_id						)
                else:
                     __lowerCAmelCase             =       len(__UpperCamelCase						)

                __lowerCAmelCase             =       self._get_best_spans(
                    start_logits=start_logits[doc_id][passage_offset:sequence_len]       ,    end_logits=end_logits[doc_id][passage_offset:sequence_len]       ,    max_answer_length=__UpperCamelCase       ,    top_spans=__UpperCamelCase       ,    )
                for start_index, end_index in best_spans:
                     start_index += passage_offset
                     end_index += passage_offset
                     nbest_spans_predictions.append(
                         DPRSpanPrediction(
                             span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]       ,    relevance_score=relevance_logits[doc_id]       ,    doc_id=__UpperCamelCase       ,    start_index=__UpperCamelCase       ,    end_index=__UpperCamelCase       ,    text=self.decode(sequence_ids[start_index : end_index + 1]						)       ,    )						)
                if len(__UpperCamelCase						) >= num_spans:
                     break
           return nbest_spans_predictions[:num_spans]


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    )->						List[DPRSpanPrediction]:
           __lowerCAmelCase             =       []
           for start_index, start_score in enumerate(__UpperCamelCase						):
                for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]						):
                     scores.append(((start_index, start_index + answer_length), start_score + end_score)						)
           __lowerCAmelCase             =       sorted(__UpperCamelCase       ,    key=lambda __UpperCamelCase						: x[1]       ,    reverse=__UpperCamelCase						)
           __lowerCAmelCase             =       []
           for (start_index, end_index), score in scores:
                assert start_index <= end_index, F"""Wrong span indices: [{start_index}:{end_index}]"""
                __lowerCAmelCase             =       end_index - start_index + 1
                assert length <= max_answer_length, F"""Span is too long: {length} > {max_answer_length}"""
                if any(
                    start_index <= prev_start_index <= prev_end_index <= end_index
                    or prev_start_index <= start_index <= end_index <= prev_end_index
                    for (prev_start_index, prev_end_index) in chosen_span_intervals						):
                     continue
                chosen_span_intervals.append((start_index, end_index)						)

                if len(__UpperCamelCase						) == top_spans:
                     break
           return chosen_span_intervals



@add_end_docstrings(a_      )
class 				_UpperCamelCase      (a_				,		a_      ):
      snake_case_	=				VOCAB_FILES_NAMES
      snake_case_	=				READER_PRETRAINED_VOCAB_FILES_MAP
      snake_case_	=				READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case_	=				READER_PRETRAINED_INIT_CONFIGURATION
      snake_case_	=				["""input_ids""", """attention_mask"""]
      snake_case_	=				DPRReaderTokenizer






