

"""simple docstring"""




import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
						from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
						from tensorflow.keras.optimizers import Adam





class  lowerCamelCase    (tf.keras.optimizers.schedules.LearningRateSchedule      ):

			def __init__(       self					:    List[Any]     ,					__UpperCAmelCase					:    float     ,					__UpperCAmelCase					:    Callable     ,					__UpperCAmelCase					:    int     ,					__UpperCAmelCase					:    float = 1.0     ,					__UpperCAmelCase					:    str = None     ,					) ->     List[str]:
						super().__init__()
						SCREAMING_SNAKE_CASE__										=   initial_learning_rate
						SCREAMING_SNAKE_CASE__										=   warmup_steps
						SCREAMING_SNAKE_CASE__										=   power
						SCREAMING_SNAKE_CASE__										=   decay_schedule_fn
						SCREAMING_SNAKE_CASE__										=   name

			def __call__(       self					:    List[Any]     ,					__UpperCAmelCase					:    List[str]							) ->     List[str]:
						with tf.name_scope(self.name or """WarmUp"""							) as name:
									# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
									# learning rate will be `global_step/num_warmup_steps * init_lr`.
									SCREAMING_SNAKE_CASE__										=   tf.cast(__UpperCAmelCase     ,					tf.floataa							)
									SCREAMING_SNAKE_CASE__										=   tf.cast(self.warmup_steps     ,					tf.floataa							)
									SCREAMING_SNAKE_CASE__										=   global_step_float / warmup_steps_float
									SCREAMING_SNAKE_CASE__										=   self.initial_learning_rate * tf.math.pow(__UpperCAmelCase     ,					self.power							)
									return tf.cond(
									    global_step_float < warmup_steps_float     ,					lambda: warmup_learning_rate     ,					lambda: self.decay_schedule_fn(step - self.warmup_steps							)     ,					name=__UpperCAmelCase     ,					)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Any:
						return {
						    "initial_learning_rate": self.initial_learning_rate,
						    "decay_schedule_fn": self.decay_schedule_fn,
						    "warmup_steps": self.warmup_steps,
						    "power": self.power,
						    "name": self.name,
						}

def 						A							(  snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = 0.0		,					snake_case__ = 0.9		,					snake_case__ = 0.9_99		,					snake_case__ = 1e-8		,					snake_case__ = None		,					snake_case__ = None		,					snake_case__ = 0.0		,					snake_case__ = 1.0		,					snake_case__ = None		,					):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   tf.keras.optimizers.schedules.PolynomialDecay(
			    initial_learning_rate=snake_case__		,					decay_steps=num_train_steps - num_warmup_steps		,					end_learning_rate=init_lr * min_lr_ratio		,					power=snake_case__		,					)
			if num_warmup_steps:
						SCREAMING_SNAKE_CASE__										=   WarmUp(
						    initial_learning_rate=snake_case__		,					decay_schedule_fn=snake_case__		,					warmup_steps=snake_case__		,					)
			if weight_decay_rate > 0.0:
						SCREAMING_SNAKE_CASE__										=   AdamWeightDecay(
						    learning_rate=snake_case__		,					weight_decay_rate=snake_case__		,					beta_a=snake_case__		,					beta_a=snake_case__		,					epsilon=snake_case__		,					clipnorm=snake_case__		,					global_clipnorm=snake_case__		,					exclude_from_weight_decay=["""LayerNorm""", """layer_norm""", """bias"""]		,					include_in_weight_decay=snake_case__		,					)
			else:
						SCREAMING_SNAKE_CASE__										=   tf.keras.optimizers.Adam(
						    learning_rate=snake_case__		,					beta_a=snake_case__		,					beta_a=snake_case__		,					epsilon=snake_case__		,					clipnorm=snake_case__		,					global_clipnorm=snake_case__		,					)
			# We return the optimizer and the LR scheduler in order to better track the
			# evolution of the LR independently of the optimizer.
			return optimizer, lr_schedule





class  lowerCamelCase    (A__      ):

			def __init__(       self					:    Tuple     ,					__UpperCAmelCase					:    Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001     ,					__UpperCAmelCase					:    float = 0.9     ,					__UpperCAmelCase					:    float = 0.999     ,					__UpperCAmelCase					:    float = 1e-7     ,					__UpperCAmelCase					:    bool = False     ,					__UpperCAmelCase					:    float = 0.0     ,					__UpperCAmelCase					:    Optional[List[str]] = None     ,					__UpperCAmelCase					:    Optional[List[str]] = None     ,					__UpperCAmelCase					:    str = "AdamWeightDecay"     ,					**__UpperCAmelCase					:    Optional[int]     ,					) ->     List[str]:
						super().__init__(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					**__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   weight_decay_rate
						SCREAMING_SNAKE_CASE__										=   include_in_weight_decay
						SCREAMING_SNAKE_CASE__										=   exclude_from_weight_decay

			@classmethod
			def        SCREAMING_SNAKE_CASE					(       cls					:    Dict     ,					__UpperCAmelCase					:    Dict							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   {"""WarmUp""": WarmUp}
						return super(__UpperCAmelCase     ,					cls							).from_config(__UpperCAmelCase     ,					custom_objects=__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    int							) ->     Union[str, Any]:
						super(__UpperCAmelCase     ,					self							)._prepare_local(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tf.constant(
						    self.weight_decay_rate     ,					name="""adam_weight_decay_rate"""							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int     ,					__UpperCAmelCase					:    List[str]     ,					__UpperCAmelCase					:    Union[str, Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self._do_use_weight_decay(var.name							)
						if do_decay:
									return var.assign_sub(
									    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["""weight_decay_rate"""]     ,					use_locking=self._use_locking     ,					)
						return tf.no_op()

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]     ,					__UpperCAmelCase					:    Optional[Any]     ,					__UpperCAmelCase					:    Dict=None     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   list(zip(*__UpperCAmelCase							)							)
						return super(__UpperCAmelCase     ,					self							).apply_gradients(zip(__UpperCAmelCase     ,					__UpperCAmelCase							)     ,					name=__UpperCAmelCase     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int     ,					__UpperCAmelCase					:    Tuple     ,					__UpperCAmelCase					:    Optional[int]     ,					__UpperCAmelCase					:    str							) ->     Tuple:
						if apply_state is None:
									return self._decayed_lr_t[var_dtype], {}

						SCREAMING_SNAKE_CASE__										=   apply_state or {}
						SCREAMING_SNAKE_CASE__										=   apply_state.get((var_device, var_dtype)							)
						if coefficients is None:
									SCREAMING_SNAKE_CASE__										=   self._fallback_apply_state(__UpperCAmelCase     ,					__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   coefficients

						return coefficients["lr_t"], {"apply_state": apply_state}

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    str     ,					__UpperCAmelCase					:    int     ,					__UpperCAmelCase					:    str=None							) ->     Tuple:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self._get_lr(var.device     ,					var.dtype.base_dtype     ,					__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   self._decay_weights_op(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase							)
						with tf.control_dependencies([decay]							):
									return super(__UpperCAmelCase     ,					self							)._resource_apply_dense(__UpperCAmelCase     ,					__UpperCAmelCase     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    str     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Optional[Any]=None							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self._get_lr(var.device     ,					var.dtype.base_dtype     ,					__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   self._decay_weights_op(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase							)
						with tf.control_dependencies([decay]							):
									return super(__UpperCAmelCase     ,					self							)._resource_apply_sparse(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   super().get_config()
						config.update({"""weight_decay_rate""": self.weight_decay_rate}							)
						return config





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						if self.weight_decay_rate == 0:
									return False

						if self._include_in_weight_decay:
									for r in self._include_in_weight_decay:
												if re.search(__UpperCAmelCase     ,					__UpperCAmelCase							) is not None:
															return True

						if self._exclude_from_weight_decay:
									for r in self._exclude_from_weight_decay:
												if re.search(__UpperCAmelCase     ,					__UpperCAmelCase							) is not None:
															return False
						return True





class  lowerCamelCase    (A__      ):

			def __init__(       self					:    Any							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   []
						SCREAMING_SNAKE_CASE__										=   None

			@property
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[Any]:
						if self._accum_steps is None:
									SCREAMING_SNAKE_CASE__										=   tf.Variable(
									    tf.constant(0     ,					dtype=tf.intaa							)     ,					trainable=__UpperCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)

						return self._accum_steps.value()

			@property
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						if not self._gradients:
									raise ValueError("""The accumulator should be called first to initialize the gradients"""							)
						return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]

			def __call__(       self					:    Any     ,					__UpperCAmelCase					:    str							) ->     Optional[int]:
						if not self._gradients:
									SCREAMING_SNAKE_CASE__										=   self.step  # Create the step variable.
									self._gradients.extend(
									    [
									        tf.Variable(
									            tf.zeros_like(__UpperCAmelCase							)     ,					trainable=__UpperCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)
									        if gradient is not None
									        else gradient
									        for gradient in gradients
									    ]							)
						if len(__UpperCAmelCase							) != len(self._gradients							):
									raise ValueError(F"""Expected {len(self._gradients							)} gradients, but got {len(__UpperCAmelCase							)}"""							)

						for accum_gradient, gradient in zip(self._gradients     ,					__UpperCAmelCase							):
									if accum_gradient is not None and gradient is not None:
												accum_gradient.assign_add(__UpperCAmelCase							)

						self._accum_steps.assign_add(1							)





			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Any:
						if not self._gradients:
									return
						self._accum_steps.assign(0							)
						for gradient in self._gradients:
									if gradient is not None:
												gradient.assign(tf.zeros_like(__UpperCAmelCase							)							)





"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_				:      List[Any]           =			logging.get_logger(__name__)

A_				:      Optional[int]           =			{
    "facebook/dpr-ctx_encoder-single-nq-base": (
        "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-question_encoder-single-nq-base": (
        "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-reader-single-nq-base": (
        "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-ctx_encoder-multiset-base": (
        "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json"
    ),
    "facebook/dpr-question_encoder-multiset-base": (
        "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json"
    ),
    "facebook/dpr-reader-multiset-base": (
        "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json"
    ),
}





class  lowerCamelCase    (A__      ):
			lowerCamelCase__ :		Optional[int]       		=				'dpr'

			def __init__(       self					:    Optional[int]     ,					__UpperCAmelCase					:    List[str]=3_0_5_2_2     ,					__UpperCAmelCase					:    int=7_6_8     ,					__UpperCAmelCase					:    Optional[Any]=1_2     ,					__UpperCAmelCase					:    Dict=1_2     ,					__UpperCAmelCase					:    Any=3_0_7_2     ,					__UpperCAmelCase					:    int="gelu"     ,					__UpperCAmelCase					:    int=0.1     ,					__UpperCAmelCase					:    Optional[Any]=0.1     ,					__UpperCAmelCase					:    Any=5_1_2     ,					__UpperCAmelCase					:    Union[str, Any]=2     ,					__UpperCAmelCase					:    Any=0.02     ,					__UpperCAmelCase					:    Dict=1e-12     ,					__UpperCAmelCase					:    Optional[Any]=0     ,					__UpperCAmelCase					:    Union[str, Any]="absolute"     ,					__UpperCAmelCase					:    int = 0     ,					**__UpperCAmelCase					:    Dict     ,					) ->     Tuple:
						super().__init__(pad_token_id=__UpperCAmelCase     ,					**__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   vocab_size
						SCREAMING_SNAKE_CASE__										=   hidden_size
						SCREAMING_SNAKE_CASE__										=   num_hidden_layers
						SCREAMING_SNAKE_CASE__										=   num_attention_heads
						SCREAMING_SNAKE_CASE__										=   hidden_act
						SCREAMING_SNAKE_CASE__										=   intermediate_size
						SCREAMING_SNAKE_CASE__										=   hidden_dropout_prob
						SCREAMING_SNAKE_CASE__										=   attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__										=   max_position_embeddings
						SCREAMING_SNAKE_CASE__										=   type_vocab_size
						SCREAMING_SNAKE_CASE__										=   initializer_range
						SCREAMING_SNAKE_CASE__										=   layer_norm_eps
						SCREAMING_SNAKE_CASE__										=   projection_dim
						SCREAMING_SNAKE_CASE__										=   position_embedding_type


