
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings





@maybe_allow_in_graph
class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case       ,					_snake_case       ,					_snake_case=0.0       ,					_snake_case = None       ,					_snake_case = "geglu"       ,					_snake_case = None       ,					_snake_case = False       ,					_snake_case = False       ,					_snake_case = False       ,					_snake_case = False       ,					_snake_case = True       ,					_snake_case = "layer_norm"       ,					_snake_case = False       ,					):
        super().__init__()
        _UpperCAmelCase 				=only_cross_attention

        _UpperCAmelCase 				=(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
        _UpperCAmelCase 				=(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

        if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
         raise ValueError(
             F"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
             F" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."	)

        # Define 3 blocks. Each block has its own normalization layer.
        # 1. Self-Attn
        if self.use_ada_layer_norm:
         _UpperCAmelCase 				=AdaLayerNorm(__a       ,					__a	)
        elif self.use_ada_layer_norm_zero:
         _UpperCAmelCase 				=AdaLayerNormZero(__a       ,					__a	)
        else:
         _UpperCAmelCase 				=nn.LayerNorm(__a       ,					elementwise_affine=__a	)
        _UpperCAmelCase 				=Attention(
            query_dim=__a       ,					heads=__a       ,					dim_head=__a       ,					dropout=__a       ,					bias=__a       ,					cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,					upcast_attention=__a       ,					)

        # 2. Cross-Attn
        if cross_attention_dim is not None or double_self_attention:
         # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
         # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
         # the second cross attention block.
         _UpperCAmelCase 				=(
             AdaLayerNorm(__a       ,					__a	)
             if self.use_ada_layer_norm
             else nn.LayerNorm(__a       ,					elementwise_affine=__a	)
         )
         _UpperCAmelCase 				=Attention(
             query_dim=__a       ,					cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,					heads=__a       ,					dim_head=__a       ,					dropout=__a       ,					bias=__a       ,					upcast_attention=__a       ,					)  # is self-attn if encoder_hidden_states is none
        else:
         _UpperCAmelCase 				=None
         _UpperCAmelCase 				=None

        # 3. Feed-forward
        _UpperCAmelCase 				=nn.LayerNorm(__a       ,					elementwise_affine=__a	)
        _UpperCAmelCase 				=FeedForward(__a       ,					dropout=__a       ,					activation_fn=__a       ,					final_dropout=__a	)

        # let chunk size default to None
        _UpperCAmelCase 				=None
        _UpperCAmelCase 				=0
       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case       ,					_snake_case	):
        # Sets chunk feed-forward
        _UpperCAmelCase 				=chunk_size
        _UpperCAmelCase 				=dim

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case       ,					_snake_case = None       ,					_snake_case = None       ,					_snake_case = None       ,					_snake_case = None       ,					_snake_case = None       ,					_snake_case = None       ,					):
        # Notice that normalization is always applied before the real computation in the following blocks.
        # 1. Self-Attention
        if self.use_ada_layer_norm:
         _UpperCAmelCase 				=self.norma(__a       ,					__a	)
        elif self.use_ada_layer_norm_zero:
         _UpperCAmelCase 				=self.norma(
             __a       ,					__a       ,					__a       ,					hidden_dtype=hidden_states.dtype	)
        else:
         _UpperCAmelCase 				=self.norma(__a	)

        _UpperCAmelCase 				=cross_attention_kwargs if cross_attention_kwargs is not None else {}

        _UpperCAmelCase 				=self.attna(
            __a       ,					encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,					attention_mask=__a       ,					**__a       ,					)
        if self.use_ada_layer_norm_zero:
         _UpperCAmelCase 				=gate_msa.unsqueeze(1	) * attn_output
        _UpperCAmelCase 				=attn_output + hidden_states

        # 2. Cross-Attention
        if self.attna is not None:
         _UpperCAmelCase 				=(
             self.norma(__a       ,					__a	) if self.use_ada_layer_norm else self.norma(__a	)
         )

         _UpperCAmelCase 				=self.attna(
             __a       ,					encoder_hidden_states=__a       ,					attention_mask=__a       ,					**__a       ,					)
         _UpperCAmelCase 				=attn_output + hidden_states

        # 3. Feed-forward
        _UpperCAmelCase 				=self.norma(__a	)

        if self.use_ada_layer_norm_zero:
         _UpperCAmelCase 				=norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

        if self._chunk_size is not None:
         # "feed_forward_chunk_size" can be used to save memory
         if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
          raise ValueError(
              F"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."	)

         _UpperCAmelCase 				=norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
         _UpperCAmelCase 				=torch.cat(
             [self.ff(__a	) for hid_slice in norm_hidden_states.chunk(__a       ,					dim=self._chunk_dim	)]       ,					dim=self._chunk_dim       ,					)
        else:
         _UpperCAmelCase 				=self.ff(__a	)

        if self.use_ada_layer_norm_zero:
         _UpperCAmelCase 				=gate_mlp.unsqueeze(1	) * ff_output

        _UpperCAmelCase 				=ff_output + hidden_states

        return hidden_states







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case = None       ,					_snake_case = 4       ,					_snake_case = 0.0       ,					_snake_case = "geglu"       ,					_snake_case = False       ,					):
        super().__init__()
        _UpperCAmelCase 				=int(dim * mult	)
        _UpperCAmelCase 				=dim_out if dim_out is not None else dim

        if activation_fn == "gelu":
         _UpperCAmelCase 				=GELU(__a       ,					__a	)
        if activation_fn == "gelu-approximate":
         _UpperCAmelCase 				=GELU(__a       ,					__a       ,					approximate="tanh"	)
        elif activation_fn == "geglu":
         _UpperCAmelCase 				=GEGLU(__a       ,					__a	)
        elif activation_fn == "geglu-approximate":
         _UpperCAmelCase 				=ApproximateGELU(__a       ,					__a	)

        _UpperCAmelCase 				=nn.ModuleList([]	)
        # project in
        self.net.append(__a	)
        # project dropout
        self.net.append(nn.Dropout(__a	)	)
        # project out
        self.net.append(nn.Linear(__a       ,					__a	)	)
        # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
        if final_dropout:
         self.net.append(nn.Dropout(__a	)	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        for module in self.net:
         _UpperCAmelCase 				=module(__a	)
        return hidden_states







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case       ,					_snake_case = "none"	):
        super().__init__()
        _UpperCAmelCase 				=nn.Linear(__a       ,					__a	)
        _UpperCAmelCase 				=approximate
       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        if gate.device.type != "mps":
         return F.gelu(__a       ,					approximate=self.approximate	)
        # mps: gelu is not implemented for float16
        return F.gelu(gate.to(dtype=torch.floataa	)       ,					approximate=self.approximate	).to(dtype=gate.dtype	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        _UpperCAmelCase 				=self.proj(__a	)
        _UpperCAmelCase 				=self.gelu(__a	)
        return hidden_states







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case	):
        super().__init__()
        _UpperCAmelCase 				=nn.Linear(__a       ,					dim_out * 2	)
       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        if gate.device.type != "mps":
         return F.gelu(__a	)
        # mps: gelu is not implemented for float16
        return F.gelu(gate.to(dtype=torch.floataa	)	).to(dtype=gate.dtype	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        _UpperCAmelCase 				=self.proj(__a	).chunk(2       ,					dim=-1	)
        return hidden_states * self.gelu(__a	)







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case	):
        super().__init__()
        _UpperCAmelCase 				=nn.Linear(__a       ,					__a	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case	):
        _UpperCAmelCase 				=self.proj(__a	)
        return x * torch.sigmoid(1.702 * x	)







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case	):
        super().__init__()
        _UpperCAmelCase 				=nn.Embedding(__a       ,					__a	)
        _UpperCAmelCase 				=nn.SiLU()
        _UpperCAmelCase 				=nn.Linear(__a       ,					embedding_dim * 2	)
        _UpperCAmelCase 				=nn.LayerNorm(__a       ,					elementwise_affine=__a	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case       ,					_snake_case	):
        _UpperCAmelCase 				=self.linear(self.silu(self.emb(__a	)	)	)
        _UpperCAmelCase 				=torch.chunk(__a       ,					2	)
        _UpperCAmelCase 				=self.norm(__a	) * (1 + scale) + shift
        return x







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case	):
        super().__init__()

        _UpperCAmelCase 				=CombinedTimestepLabelEmbeddings(__a       ,					__a	)

        _UpperCAmelCase 				=nn.SiLU()
        _UpperCAmelCase 				=nn.Linear(__a       ,					6 * embedding_dim       ,					bias=__a	)
        _UpperCAmelCase 				=nn.LayerNorm(__a       ,					elementwise_affine=__a       ,					eps=1E-6	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case       ,					_snake_case       ,					_snake_case       ,					_snake_case=None	):
        _UpperCAmelCase 				=self.linear(self.silu(self.emb(__a       ,					__a       ,					hidden_dtype=__a	)	)	)
        _UpperCAmelCase 				=emb.chunk(6       ,					dim=1	)
        _UpperCAmelCase 				=self.norm(__a	) * (1 + scale_msa[:, None]) + shift_msa[:, None]
        return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 						_a   (						nn.Module       ):

       """simple docstring"""





       def __init__(					self       ,					_snake_case       ,					_snake_case       ,					_snake_case       ,					_snake_case = None       ,					_snake_case = 1E-5	):
        super().__init__()
        _UpperCAmelCase 				=num_groups
        _UpperCAmelCase 				=eps

        if act_fn is None:
         _UpperCAmelCase 				=None
        else:
         _UpperCAmelCase 				=get_activation(__a	)

        _UpperCAmelCase 				=nn.Linear(__a       ,					out_dim * 2	)

       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case       ,					_snake_case	):
        if self.act:
         _UpperCAmelCase 				=self.act(__a	)
        _UpperCAmelCase 				=self.linear(__a	)
        _UpperCAmelCase 				=emb[:, :, None, None]
        _UpperCAmelCase 				=emb.chunk(2       ,					dim=1	)

        _UpperCAmelCase 				=F.group_norm(__a       ,					self.num_groups       ,					eps=self.eps	)
        _UpperCAmelCase 				=x * (1 + scale) + shift
        return x









from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__				:	List[Any]					       =					logging.get_logger(__name__)

snake_case__				:	str					       =					{
    'facebook/dpr-ctx_encoder-single-nq-base': (
        'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-question_encoder-single-nq-base': (
        'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-reader-single-nq-base': (
        'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-ctx_encoder-multiset-base': (
        'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json'
    ),
    'facebook/dpr-question_encoder-multiset-base': (
        'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json'
    ),
    'facebook/dpr-reader-multiset-base': (
        'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json'
    ),
}





class   SCREAMING_SNAKE_CASE_			(a__						):




       '''simple docstring'''



       _a   							=							"dpr"



       def __init__(						self				:				Union[str, Any]		,				__a				:				Optional[Any]=30_522		,				__a				:				List[Any]=768		,				__a				:				List[Any]=12		,				__a				:				Dict=12		,				__a				:				Union[str, Any]=3_072		,				__a				:				Any="gelu"		,				__a				:				Any=0.1		,				__a				:				Any=0.1		,				__a				:				Tuple=512		,				__a				:				int=2		,				__a				:				Optional[Any]=0.02		,				__a				:				List[Any]=1e-12		,				__a				:				int=0		,				__a				:				int="absolute"		,				__a				:				int = 0		,				**__a				:				Optional[int]		,				)					->Tuple:
          super().__init__(pad_token_id=__a		,				**__a							)

          lowerCamelCase_						:   str		= vocab_size
          lowerCamelCase_						:   List[Any]		= hidden_size
          lowerCamelCase_						:   Optional[int]		= num_hidden_layers
          lowerCamelCase_						:   Optional[Any]		= num_attention_heads
          lowerCamelCase_						:   List[Any]		= hidden_act
          lowerCamelCase_						:   Optional[Any]		= intermediate_size
          lowerCamelCase_						:   List[Any]		= hidden_dropout_prob
          lowerCamelCase_						:   Any		= attention_probs_dropout_prob
          lowerCamelCase_						:   Any		= max_position_embeddings
          lowerCamelCase_						:   Tuple		= type_vocab_size
          lowerCamelCase_						:   Dict		= initializer_range
          lowerCamelCase_						:   List[Any]		= layer_norm_eps
          lowerCamelCase_						:   List[str]		= projection_dim
          lowerCamelCase_						:   Optional[int]		= position_embedding_type





