


"""simple docstring"""
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE=0.0      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = "geglu"      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = False      ,   SCREAMING_SNAKE_CASE = False      ,   SCREAMING_SNAKE_CASE = False      ,   SCREAMING_SNAKE_CASE = False      ,   SCREAMING_SNAKE_CASE = True      ,   SCREAMING_SNAKE_CASE = "layer_norm"      ,   SCREAMING_SNAKE_CASE = False      ,   )  ->  Dict:
      super().__init__()
      _lowerCamelCase	:					Tuple   =		only_cross_attention

      _lowerCamelCase	:					int   =		(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
      _lowerCamelCase	:					Any   =		(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

      if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
         raise ValueError(
             F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
             F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.')

      # Define 3 blocks. Each block has its own normalization layer.
      # 1. Self-Attn
      if self.use_ada_layer_norm:
         _lowerCamelCase	:					List[Any]   =		AdaLayerNorm(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      elif self.use_ada_layer_norm_zero:
         _lowerCamelCase	:					str   =		AdaLayerNormZero(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      else:
         _lowerCamelCase	:					Tuple   =		nn.LayerNorm(SCREAMING_SNAKE_CASE      ,   elementwise_affine=SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Union[str, Any]   =		Attention(
          query_dim=SCREAMING_SNAKE_CASE      ,   heads=SCREAMING_SNAKE_CASE      ,   dim_head=SCREAMING_SNAKE_CASE      ,   dropout=SCREAMING_SNAKE_CASE      ,   bias=SCREAMING_SNAKE_CASE      ,   cross_attention_dim=cross_attention_dim if only_cross_attention else None      ,   upcast_attention=SCREAMING_SNAKE_CASE      ,   )

      # 2. Cross-Attn
      if cross_attention_dim is not None or double_self_attention:
         # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
         # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
         # the second cross attention block.
         _lowerCamelCase	:					Dict   =		(
             AdaLayerNorm(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
             if self.use_ada_layer_norm
             else nn.LayerNorm(SCREAMING_SNAKE_CASE      ,   elementwise_affine=SCREAMING_SNAKE_CASE)
         )
         _lowerCamelCase	:					Tuple   =		Attention(
             query_dim=SCREAMING_SNAKE_CASE      ,   cross_attention_dim=cross_attention_dim if not double_self_attention else None      ,   heads=SCREAMING_SNAKE_CASE      ,   dim_head=SCREAMING_SNAKE_CASE      ,   dropout=SCREAMING_SNAKE_CASE      ,   bias=SCREAMING_SNAKE_CASE      ,   upcast_attention=SCREAMING_SNAKE_CASE      ,   )  # is self-attn if encoder_hidden_states is none
      else:
         _lowerCamelCase	:					Any   =		None
         _lowerCamelCase	:					List[str]   =		None

      # 3. Feed-forward
      _lowerCamelCase	:					int   =		nn.LayerNorm(SCREAMING_SNAKE_CASE      ,   elementwise_affine=SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					List[Any]   =		FeedForward(SCREAMING_SNAKE_CASE      ,   dropout=SCREAMING_SNAKE_CASE      ,   activation_fn=SCREAMING_SNAKE_CASE      ,   final_dropout=SCREAMING_SNAKE_CASE)

      # let chunk size default to None
      _lowerCamelCase	:					Union[str, Any]   =		None
      _lowerCamelCase	:					str   =		0





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  Union[str, Any]:
      # Sets chunk feed-forward
      _lowerCamelCase	:					str   =		chunk_size
      _lowerCamelCase	:					Optional[int]   =		dim





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = None      ,   )  ->  str:
      # Notice that normalization is always applied before the real computation in the following blocks.
      # 1. Self-Attention
      if self.use_ada_layer_norm:
         _lowerCamelCase	:					Union[str, Any]   =		self.norma(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      elif self.use_ada_layer_norm_zero:
         _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase	:					Dict   =		self.norma(
             SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   hidden_dtype=hidden_states.dtype)
      else:
         _lowerCamelCase	:					List[Any]   =		self.norma(SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					List[Any]   =		cross_attention_kwargs if cross_attention_kwargs is not None else {}

      _lowerCamelCase	:					Any   =		self.attna(
          SCREAMING_SNAKE_CASE      ,   encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None      ,   attention_mask=SCREAMING_SNAKE_CASE      ,   **SCREAMING_SNAKE_CASE      ,   )
      if self.use_ada_layer_norm_zero:
         _lowerCamelCase	:					Optional[Any]   =		gate_msa.unsqueeze(1) * attn_output
      _lowerCamelCase	:					Tuple   =		attn_output + hidden_states

      # 2. Cross-Attention
      if self.attna is not None:
         _lowerCamelCase	:					Optional[Any]   =		(
             self.norma(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE) if self.use_ada_layer_norm else self.norma(SCREAMING_SNAKE_CASE)
         )

         _lowerCamelCase	:					Optional[Any]   =		self.attna(
             SCREAMING_SNAKE_CASE      ,   encoder_hidden_states=SCREAMING_SNAKE_CASE      ,   attention_mask=SCREAMING_SNAKE_CASE      ,   **SCREAMING_SNAKE_CASE      ,   )
         _lowerCamelCase	:					Any   =		attn_output + hidden_states

      # 3. Feed-forward
      _lowerCamelCase	:					Union[str, Any]   =		self.norma(SCREAMING_SNAKE_CASE)

      if self.use_ada_layer_norm_zero:
         _lowerCamelCase	:					Optional[int]   =		norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

      if self._chunk_size is not None:
         # "feed_forward_chunk_size" can be used to save memory
         if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
            raise ValueError(
                F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.')

         _lowerCamelCase	:					Union[str, Any]   =		norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
         _lowerCamelCase	:					int   =		torch.cat(
             [self.ff(SCREAMING_SNAKE_CASE) for hid_slice in norm_hidden_states.chunk(SCREAMING_SNAKE_CASE      ,   dim=self._chunk_dim)]      ,   dim=self._chunk_dim      ,   )
      else:
         _lowerCamelCase	:					Optional[Any]   =		self.ff(SCREAMING_SNAKE_CASE)

      if self.use_ada_layer_norm_zero:
         _lowerCamelCase	:					Any   =		gate_mlp.unsqueeze(1) * ff_output

      _lowerCamelCase	:					Tuple   =		ff_output + hidden_states

      return hidden_states







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = 4      ,   SCREAMING_SNAKE_CASE = 0.0      ,   SCREAMING_SNAKE_CASE = "geglu"      ,   SCREAMING_SNAKE_CASE = False      ,   )  ->  str:
      super().__init__()
      _lowerCamelCase	:					Optional[int]   =		int(dim * mult)
      _lowerCamelCase	:					Optional[int]   =		dim_out if dim_out is not None else dim

      if activation_fn == "gelu":
         _lowerCamelCase	:					Tuple   =		GELU(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      if activation_fn == "gelu-approximate":
         _lowerCamelCase	:					List[Any]   =		GELU(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   approximate="""tanh""")
      elif activation_fn == "geglu":
         _lowerCamelCase	:					str   =		GEGLU(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      elif activation_fn == "geglu-approximate":
         _lowerCamelCase	:					int   =		ApproximateGELU(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					Any   =		nn.ModuleList([])
      # project in
      self.net.append(SCREAMING_SNAKE_CASE)
      # project dropout
      self.net.append(nn.Dropout(SCREAMING_SNAKE_CASE))
      # project out
      self.net.append(nn.Linear(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE))
      # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
      if final_dropout:
         self.net.append(nn.Dropout(SCREAMING_SNAKE_CASE))





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  Tuple:
      for module in self.net:
         _lowerCamelCase	:					Tuple   =		module(SCREAMING_SNAKE_CASE)
      return hidden_states







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = "none")  ->  Optional[int]:
      super().__init__()
      _lowerCamelCase	:					int   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Union[str, Any]   =		approximate





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  List[Any]:
      if gate.device.type != "mps":
         return F.gelu(SCREAMING_SNAKE_CASE      ,   approximate=self.approximate)
      # mps: gelu is not implemented for float16
      return F.gelu(gate.to(dtype=torch.floataa)      ,   approximate=self.approximate).to(dtype=gate.dtype)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  int:
      _lowerCamelCase	:					Tuple   =		self.proj(SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Union[str, Any]   =		self.gelu(SCREAMING_SNAKE_CASE)
      return hidden_states







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  List[Any]:
      super().__init__()
      _lowerCamelCase	:					Dict   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   dim_out * 2)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  List[str]:
      if gate.device.type != "mps":
         return F.gelu(SCREAMING_SNAKE_CASE)
      # mps: gelu is not implemented for float16
      return F.gelu(gate.to(dtype=torch.floataa)).to(dtype=gate.dtype)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  Union[str, Any]:
      _lowerCamelCase   ,    _lowerCamelCase	:					str   =		self.proj(SCREAMING_SNAKE_CASE).chunk(2      ,   dim=-1)
      return hidden_states * self.gelu(SCREAMING_SNAKE_CASE)







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  Tuple:
      super().__init__()
      _lowerCamelCase	:					Union[str, Any]   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  Any:
      _lowerCamelCase	:					Union[str, Any]   =		self.proj(SCREAMING_SNAKE_CASE)
      return x * torch.sigmoid(1.7_02 * x)







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  Union[str, Any]:
      super().__init__()
      _lowerCamelCase	:					int   =		nn.Embedding(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Dict   =		nn.SiLU()
      _lowerCamelCase	:					Any   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   embedding_dim * 2)
      _lowerCamelCase	:					Optional[Any]   =		nn.LayerNorm(SCREAMING_SNAKE_CASE      ,   elementwise_affine=SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  List[str]:
      _lowerCamelCase	:					Tuple   =		self.linear(self.silu(self.emb(SCREAMING_SNAKE_CASE)))
      _lowerCamelCase   ,    _lowerCamelCase	:					str   =		torch.chunk(SCREAMING_SNAKE_CASE      ,   2)
      _lowerCamelCase	:					Optional[int]   =		self.norm(SCREAMING_SNAKE_CASE) * (1 + scale) + shift
      return x







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  int:
      super().__init__()

      _lowerCamelCase	:					Any   =		CombinedTimestepLabelEmbeddings(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					Any   =		nn.SiLU()
      _lowerCamelCase	:					Optional[Any]   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   6 * embedding_dim      ,   bias=SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					int   =		nn.LayerNorm(SCREAMING_SNAKE_CASE      ,   elementwise_affine=SCREAMING_SNAKE_CASE      ,   eps=1e-6)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE=None)  ->  List[str]:
      _lowerCamelCase	:					Optional[int]   =		self.linear(self.silu(self.emb(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   hidden_dtype=SCREAMING_SNAKE_CASE)))
      _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase	:					List[Any]   =		emb.chunk(6      ,   dim=1)
      _lowerCamelCase	:					List[Any]   =		self.norm(SCREAMING_SNAKE_CASE) * (1 + scale_msa[:, None]) + shift_msa[:, None]
      return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 	lowercase__				(					nn.Module				):





   def __init__(							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = None      ,   SCREAMING_SNAKE_CASE = 1e-5)  ->  Any:
      super().__init__()
      _lowerCamelCase	:					Optional[int]   =		num_groups
      _lowerCamelCase	:					Optional[int]   =		eps

      if act_fn is None:
         _lowerCamelCase	:					Dict   =		None
      else:
         _lowerCamelCase	:					Tuple   =		get_activation(SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					Dict   =		nn.Linear(SCREAMING_SNAKE_CASE      ,   out_dim * 2)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE)  ->  str:
      if self.act:
         _lowerCamelCase	:					int   =		self.act(SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					str   =		self.linear(SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Optional[Any]   =		emb[:, :, None, None]
      _lowerCamelCase   ,    _lowerCamelCase	:					Any   =		emb.chunk(2      ,   dim=1)

      _lowerCamelCase	:					List[str]   =		F.group_norm(SCREAMING_SNAKE_CASE      ,   self.num_groups      ,   eps=self.eps)
      _lowerCamelCase	:					Optional[Any]   =		x * (1 + scale) + shift
      return x



"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_torch_available,
    is_vision_available,
)


SCREAMING_SNAKE_CASE_            =    {'''configuration_beit''': ['''BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BeitConfig''', '''BeitOnnxConfig''']}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    ['''BeitFeatureExtractor''']
   SCREAMING_SNAKE_CASE_            =    ['''BeitImageProcessor''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    [
       '''BEIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''BeitForImageClassification''',
       '''BeitForMaskedImageModeling''',
       '''BeitForSemanticSegmentation''',
       '''BeitModel''',
       '''BeitPreTrainedModel''',
   ]


try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    [
       '''FlaxBeitForImageClassification''',
       '''FlaxBeitForMaskedImageModeling''',
       '''FlaxBeitModel''',
       '''FlaxBeitPreTrainedModel''',
   ]

if TYPE_CHECKING:
   from .configuration_beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig, BeitOnnxConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_beit import BeitFeatureExtractor
      from .image_processing_beit import BeitImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_beit import (
          BEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
          BeitForImageClassification,
          BeitForMaskedImageModeling,
          BeitForSemanticSegmentation,
          BeitModel,
          BeitPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_beit import (
          FlaxBeitForImageClassification,
          FlaxBeitForMaskedImageModeling,
          FlaxBeitModel,
          FlaxBeitPreTrainedModel,
      )


else:
   import sys

   SCREAMING_SNAKE_CASE_            =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



