from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class        snake_case__							(					nn.Module				):



   def __init__(					self : List[Any]      ,						A__ : List[Any]      ,						A__ : int      ,						A__ : int      ,						A__ : int=0.0      ,						A__ : List[str] = None      ,						A__ : Tuple = "geglu"      ,						A__ : Union[str, Any] = None      ,						A__ : List[Any] = False      ,						A__ : Any = False      ,						A__ : int = False      ,						A__ : Tuple = False      ,						A__ : List[str] = True      ,						A__ : Tuple = "layer_norm"      ,						A__ : List[str] = False      ,						)				->  Union[str, Any]:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      List[Any]  	=							only_cross_attention

         snake_case_  :      Optional[Any]  	=							(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
         snake_case_  :      List[str]  	=							(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
               raise ValueError(
                   f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
                   f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." )

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
               snake_case_  :      List[str]  	=							AdaLayerNorm(snake_case__      ,						snake_case__ )
         elif self.use_ada_layer_norm_zero:
               snake_case_  :      Dict  	=							AdaLayerNormZero(snake_case__      ,						snake_case__ )
         else:
               snake_case_  :      Any  	=							nn.LayerNorm(snake_case__      ,						elementwise_affine=snake_case__ )
         snake_case_  :      str  	=							Attention(
             query_dim=snake_case__      ,						heads=snake_case__      ,						dim_head=snake_case__      ,						dropout=snake_case__      ,						bias=snake_case__      ,						cross_attention_dim=cross_attention_dim if only_cross_attention else None      ,						upcast_attention=snake_case__      ,						)

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
               # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
               # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
               # the second cross attention block.
               snake_case_  :      List[str]  	=							(
                   AdaLayerNorm(snake_case__      ,						snake_case__ )
                   if self.use_ada_layer_norm
                   else nn.LayerNorm(snake_case__      ,						elementwise_affine=snake_case__ )
               )
               snake_case_  :      Tuple  	=							Attention(
                   query_dim=snake_case__      ,						cross_attention_dim=cross_attention_dim if not double_self_attention else None      ,						heads=snake_case__      ,						dim_head=snake_case__      ,						dropout=snake_case__      ,						bias=snake_case__      ,						upcast_attention=snake_case__      ,						)  # is self-attn if encoder_hidden_states is none
         else:
               snake_case_  :      Optional[Any]  	=							None
               snake_case_  :      str  	=							None

         # 3. Feed-forward
         snake_case_  :      Optional[int]  	=							nn.LayerNorm(snake_case__      ,						elementwise_affine=snake_case__ )
         snake_case_  :      Union[str, Any]  	=							FeedForward(snake_case__      ,						dropout=snake_case__      ,						activation_fn=snake_case__      ,						final_dropout=snake_case__ )

         # let chunk size default to None
         snake_case_  :      Any  	=							None
         snake_case_  :      Optional[int]  	=							0



   def        UpperCAmelCase__							(					self : Optional[Any]      ,						A__ : Dict      ,						A__ : Union[str, Any] )				->  List[str]:




         '''simple docstring'''




         snake_case_  :      int  	=							chunk_size
         snake_case_  :      Tuple  	=							dim



   def        UpperCAmelCase__							(					self : Union[str, Any]      ,						A__ : str      ,						A__ : Any = None      ,						A__ : Union[str, Any] = None      ,						A__ : int = None      ,						A__ : Union[str, Any] = None      ,						A__ : Union[str, Any] = None      ,						A__ : List[str] = None      ,						)				->  Union[str, Any]:




         '''simple docstring'''




         if self.use_ada_layer_norm:
               snake_case_  :      Dict  	=							self.norma(snake_case__      ,						snake_case__ )
         elif self.use_ada_layer_norm_zero:
               snake_case_  :      Any  	=							self.norma(
                   snake_case__      ,						snake_case__      ,						snake_case__      ,						hidden_dtype=hidden_states.dtype )
         else:
               snake_case_  :      Optional[Any]  	=							self.norma(snake_case__ )

         snake_case_  :      Optional[int]  	=							cross_attention_kwargs if cross_attention_kwargs is not None else {}

         snake_case_  :      int  	=							self.attna(
             snake_case__      ,						encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None      ,						attention_mask=snake_case__      ,						**snake_case__      ,						)
         if self.use_ada_layer_norm_zero:
               snake_case_  :      Optional[int]  	=							gate_msa.unsqueeze(1 ) * attn_output
         snake_case_  :      List[Any]  	=							attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
               snake_case_  :      Tuple  	=							(
                   self.norma(snake_case__      ,						snake_case__ ) if self.use_ada_layer_norm else self.norma(snake_case__ )
               )

               snake_case_  :      Union[str, Any]  	=							self.attna(
                   snake_case__      ,						encoder_hidden_states=snake_case__      ,						attention_mask=snake_case__      ,						**snake_case__      ,						)
               snake_case_  :      List[str]  	=							attn_output + hidden_states

         # 3. Feed-forward
         snake_case_  :      Optional[Any]  	=							self.norma(snake_case__ )

         if self.use_ada_layer_norm_zero:
               snake_case_  :      Tuple  	=							norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
               # "feed_forward_chunk_size" can be used to save memory
               if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                     raise ValueError(
                         f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." )

               snake_case_  :      List[Any]  	=							norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
               snake_case_  :      Union[str, Any]  	=							torch.cat(
                   [self.ff(snake_case__ ) for hid_slice in norm_hidden_states.chunk(snake_case__      ,						dim=self._chunk_dim )]      ,						dim=self._chunk_dim      ,						)
         else:
               snake_case_  :      Tuple  	=							self.ff(snake_case__ )

         if self.use_ada_layer_norm_zero:
               snake_case_  :      List[Any]  	=							gate_mlp.unsqueeze(1 ) * ff_output

         snake_case_  :      Dict  	=							ff_output + hidden_states

         return hidden_states



class        snake_case__							(					nn.Module				):



   def __init__(					self : int      ,						A__ : int      ,						A__ : Union[str, Any] = None      ,						A__ : List[str] = 4      ,						A__ : Optional[Any] = 0.0      ,						A__ : Union[str, Any] = "geglu"      ,						A__ : str = False      ,						)				->  List[Any]:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      Tuple  	=							int(dim * mult )
         snake_case_  :      List[str]  	=							dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
               snake_case_  :      Optional[Any]  	=							GELU(snake_case__      ,						snake_case__ )
         if activation_fn == "gelu-approximate":
               snake_case_  :      Tuple  	=							GELU(snake_case__      ,						snake_case__      ,						approximate="tanh" )
         elif activation_fn == "geglu":
               snake_case_  :      Dict  	=							GEGLU(snake_case__      ,						snake_case__ )
         elif activation_fn == "geglu-approximate":
               snake_case_  :      Union[str, Any]  	=							ApproximateGELU(snake_case__      ,						snake_case__ )

         snake_case_  :      Optional[Any]  	=							nn.ModuleList([] )
         # project in
         self.net.append(snake_case__ )
         # project dropout
         self.net.append(nn.Dropout(snake_case__ ) )
         # project out
         self.net.append(nn.Linear(snake_case__      ,						snake_case__ ) )
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
               self.net.append(nn.Dropout(snake_case__ ) )



   def        UpperCAmelCase__							(					self : List[Any]      ,						A__ : Tuple )				->  List[str]:




         '''simple docstring'''




         for module in self.net:
               snake_case_  :      Dict  	=							module(snake_case__ )
         return hidden_states



class        snake_case__							(					nn.Module				):



   def __init__(					self : Optional[int]      ,						A__ : Any      ,						A__ : str      ,						A__ : Dict = "none" )				->  Tuple:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      Tuple  	=							nn.Linear(snake_case__      ,						snake_case__ )
         snake_case_  :      Optional[Any]  	=							approximate



   def        UpperCAmelCase__							(					self : Tuple      ,						A__ : str )				->  List[Any]:




         '''simple docstring'''




         if gate.device.type != "mps":
               return F.gelu(snake_case__      ,						approximate=self.approximate )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa )      ,						approximate=self.approximate ).to(dtype=gate.dtype )



   def        UpperCAmelCase__							(					self : List[Any]      ,						A__ : str )				->  Optional[Any]:




         '''simple docstring'''




         snake_case_  :      str  	=							self.proj(snake_case__ )
         snake_case_  :      int  	=							self.gelu(snake_case__ )
         return hidden_states



class        snake_case__							(					nn.Module				):



   def __init__(					self : Union[str, Any]      ,						A__ : List[Any]      ,						A__ : Tuple )				->  Optional[Any]:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      Union[str, Any]  	=							nn.Linear(snake_case__      ,						dim_out * 2 )



   def        UpperCAmelCase__							(					self : Any      ,						A__ : int )				->  str:




         '''simple docstring'''




         if gate.device.type != "mps":
               return F.gelu(snake_case__ )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa ) ).to(dtype=gate.dtype )



   def        UpperCAmelCase__							(					self : Dict      ,						A__ : List[Any] )				->  Optional[Any]:




         '''simple docstring'''




         snake_case_  :      Any  	=							self.proj(snake_case__ ).chunk(2      ,						dim=-1 )
         return hidden_states * self.gelu(snake_case__ )



class        snake_case__							(					nn.Module				):



   def __init__(					self : Tuple      ,						A__ : Union[str, Any]      ,						A__ : Optional[int] )				->  Union[str, Any]:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      int  	=							nn.Linear(snake_case__      ,						snake_case__ )



   def        UpperCAmelCase__							(					self : int      ,						A__ : Dict )				->  Optional[Any]:




         '''simple docstring'''




         snake_case_  :      Optional[Any]  	=							self.proj(snake_case__ )
         return x * torch.sigmoid(1.702 * x )



class        snake_case__							(					nn.Module				):



   def __init__(					self : str      ,						A__ : int      ,						A__ : List[Any] )				->  List[Any]:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      int  	=							nn.Embedding(snake_case__      ,						snake_case__ )
         snake_case_  :      Tuple  	=							nn.SiLU()
         snake_case_  :      Tuple  	=							nn.Linear(snake_case__      ,						embedding_dim * 2 )
         snake_case_  :      Tuple  	=							nn.LayerNorm(snake_case__      ,						elementwise_affine=snake_case__ )



   def        UpperCAmelCase__							(					self : Union[str, Any]      ,						A__ : List[Any]      ,						A__ : Dict )				->  List[Any]:




         '''simple docstring'''




         snake_case_  :      Optional[Any]  	=							self.linear(self.silu(self.emb(snake_case__ ) ) )
         snake_case_  :      Dict  	=							torch.chunk(snake_case__      ,						2 )
         snake_case_  :      Tuple  	=							self.norm(snake_case__ ) * (1 + scale) + shift
         return x



class        snake_case__							(					nn.Module				):



   def __init__(					self : int      ,						A__ : int      ,						A__ : Tuple )				->  Tuple:




         '''simple docstring'''




         super().__init__()

         snake_case_  :      List[Any]  	=							CombinedTimestepLabelEmbeddings(snake_case__      ,						snake_case__ )

         snake_case_  :      int  	=							nn.SiLU()
         snake_case_  :      Dict  	=							nn.Linear(snake_case__      ,						6 * embedding_dim      ,						bias=snake_case__ )
         snake_case_  :      Optional[Any]  	=							nn.LayerNorm(snake_case__      ,						elementwise_affine=snake_case__      ,						eps=1E-6 )



   def        UpperCAmelCase__							(					self : Tuple      ,						A__ : int      ,						A__ : Optional[int]      ,						A__ : int      ,						A__ : Optional[int]=None )				->  Union[str, Any]:




         '''simple docstring'''




         snake_case_  :      Optional[Any]  	=							self.linear(self.silu(self.emb(snake_case__      ,						snake_case__      ,						hidden_dtype=snake_case__ ) ) )
         snake_case_  :      List[str]  	=							emb.chunk(6      ,						dim=1 )
         snake_case_  :      int  	=							self.norm(snake_case__ ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp



class        snake_case__							(					nn.Module				):



   def __init__(					self : Any      ,						A__ : str      ,						A__ : int      ,						A__ : List[Any]      ,						A__ : Optional[Any] = None      ,						A__ : str = 1E-5 )				->  Dict:




         '''simple docstring'''




         super().__init__()
         snake_case_  :      str  	=							num_groups
         snake_case_  :      Any  	=							eps

         if act_fn is None:
               snake_case_  :      Union[str, Any]  	=							None
         else:
               snake_case_  :      Dict  	=							get_activation(snake_case__ )

         snake_case_  :      Union[str, Any]  	=							nn.Linear(snake_case__      ,						out_dim * 2 )



   def        UpperCAmelCase__							(					self : List[Any]      ,						A__ : str      ,						A__ : Any )				->  List[Any]:




         '''simple docstring'''




         if self.act:
               snake_case_  :      Optional[int]  	=							self.act(snake_case__ )
         snake_case_  :      int  	=							self.linear(snake_case__ )
         snake_case_  :      int  	=							emb[:, :, None, None]
         snake_case_  :      Optional[int]  	=							emb.chunk(2      ,						dim=1 )

         snake_case_  :      Dict  	=							F.group_norm(snake_case__      ,						self.num_groups      ,						eps=self.eps )
         snake_case_  :      Tuple  	=							x * (1 + scale) + shift
         return x










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_							=				{
    """configuration_maskformer""": ["""MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MaskFormerConfig"""],
    """configuration_maskformer_swin""": ["""MaskFormerSwinConfig"""],
}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase_							=				["""MaskFormerFeatureExtractor"""]
							UpperCAmelCase_							=				["""MaskFormerImageProcessor"""]


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase_							=				[
							    """MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """MaskFormerForInstanceSegmentation""",
							    """MaskFormerModel""",
							    """MaskFormerPreTrainedModel""",
							]
							UpperCAmelCase_							=				[
							    """MaskFormerSwinBackbone""",
							    """MaskFormerSwinModel""",
							    """MaskFormerSwinPreTrainedModel""",
							]

if TYPE_CHECKING:
							from .configuration_maskformer import MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskFormerConfig
							from .configuration_maskformer_swin import MaskFormerSwinConfig

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_maskformer import MaskFormerFeatureExtractor
														from .image_processing_maskformer import MaskFormerImageProcessor
							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_maskformer import (
														    MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    MaskFormerForInstanceSegmentation,
														    MaskFormerModel,
														    MaskFormerPreTrainedModel,
														)
														from .modeling_maskformer_swin import (
														    MaskFormerSwinBackbone,
														    MaskFormerSwinModel,
														    MaskFormerSwinPreTrainedModel,
														)


else:
							import sys

							UpperCAmelCase_							=				_LazyModule(__name__, globals()["""__file__"""], _import_structure)