





"""simple docstring"""





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      List[Any]						,				__lowercase       :      int						,				__lowercase       :      int						,				__lowercase       :      int						,				__lowercase       :      Optional[int]=0.0						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      str = "geglu"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      bool = False						,				__lowercase       :      bool = False						,				__lowercase       :      bool = False						,				__lowercase       :      bool = False						,				__lowercase       :      bool = True						,				__lowercase       :      str = "layer_norm"						,				__lowercase       :      bool = False						,				):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						str       =						only_cross_attention

             __UpperCAmelCase    :						List[Any]       =						(num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
             __UpperCAmelCase    :						Any       =						(num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

             if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
                    raise ValueError(
                        f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
                        f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.''' )

             # Define 3 blocks. Each block has its own normalization layer.
             # 1. Self-Attn
             if self.use_ada_layer_norm:
                    __UpperCAmelCase    :						Dict       =						AdaLayerNorm(UpperCAmelCase_						,				UpperCAmelCase_ )
             elif self.use_ada_layer_norm_zero:
                    __UpperCAmelCase    :						Optional[Any]       =						AdaLayerNormZero(UpperCAmelCase_						,				UpperCAmelCase_ )
             else:
                    __UpperCAmelCase    :						str       =						nn.LayerNorm(UpperCAmelCase_						,				elementwise_affine=UpperCAmelCase_ )
             __UpperCAmelCase    :						Tuple       =						Attention(
                 query_dim=UpperCAmelCase_						,				heads=UpperCAmelCase_						,				dim_head=UpperCAmelCase_						,				dropout=UpperCAmelCase_						,				bias=UpperCAmelCase_						,				cross_attention_dim=cross_attention_dim if only_cross_attention else None						,				upcast_attention=UpperCAmelCase_						,				)

             # 2. Cross-Attn
             if cross_attention_dim is not None or double_self_attention:
                    # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
                    # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
                    # the second cross attention block.
                    __UpperCAmelCase    :						Tuple       =						(
                        AdaLayerNorm(UpperCAmelCase_						,				UpperCAmelCase_ )
                        if self.use_ada_layer_norm
                        else nn.LayerNorm(UpperCAmelCase_						,				elementwise_affine=UpperCAmelCase_ )
                    )
                    __UpperCAmelCase    :						int       =						Attention(
                        query_dim=UpperCAmelCase_						,				cross_attention_dim=cross_attention_dim if not double_self_attention else None						,				heads=UpperCAmelCase_						,				dim_head=UpperCAmelCase_						,				dropout=UpperCAmelCase_						,				bias=UpperCAmelCase_						,				upcast_attention=UpperCAmelCase_						,				)  # is self-attn if encoder_hidden_states is none
             else:
                    __UpperCAmelCase    :						Union[str, Any]       =						None
                    __UpperCAmelCase    :						str       =						None

             # 3. Feed-forward
             __UpperCAmelCase    :						Dict       =						nn.LayerNorm(UpperCAmelCase_						,				elementwise_affine=UpperCAmelCase_ )
             __UpperCAmelCase    :						Optional[Any]       =						FeedForward(UpperCAmelCase_						,				dropout=UpperCAmelCase_						,				activation_fn=UpperCAmelCase_						,				final_dropout=UpperCAmelCase_ )

             # let chunk size default to None
             __UpperCAmelCase    :						int       =						None
             __UpperCAmelCase    :						List[Any]       =						0

      def       A_					(							self       :      Any						,				__lowercase       :      Optional[int]						,				__lowercase       :      int ):


             '''simple docstring'''




             __UpperCAmelCase    :						str       =						chunk_size
             __UpperCAmelCase    :						Optional[Any]       =						dim







      def       A_					(							self       :      int						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.LongTensor] = None						,				__lowercase       :      Dict[str, Any] = None						,				__lowercase       :      Optional[torch.LongTensor] = None						,				):


             '''simple docstring'''




             if self.use_ada_layer_norm:
                    __UpperCAmelCase    :						int       =						self.norma(UpperCAmelCase_						,				UpperCAmelCase_ )
             elif self.use_ada_layer_norm_zero:
                    __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						self.norma(
                        UpperCAmelCase_						,				UpperCAmelCase_						,				UpperCAmelCase_						,				hidden_dtype=hidden_states.dtype )
             else:
                    __UpperCAmelCase    :						Any       =						self.norma(UpperCAmelCase_ )

             __UpperCAmelCase    :						Dict       =						cross_attention_kwargs if cross_attention_kwargs is not None else {}

             __UpperCAmelCase    :						str       =						self.attna(
                 UpperCAmelCase_						,				encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None						,				attention_mask=UpperCAmelCase_						,				**UpperCAmelCase_						,				)
             if self.use_ada_layer_norm_zero:
                    __UpperCAmelCase    :						Any       =						gate_msa.unsqueeze(1 ) * attn_output
             __UpperCAmelCase    :						int       =						attn_output + hidden_states

             # 2. Cross-Attention
             if self.attna is not None:
                    __UpperCAmelCase    :						Dict       =						(
                        self.norma(UpperCAmelCase_						,				UpperCAmelCase_ ) if self.use_ada_layer_norm else self.norma(UpperCAmelCase_ )
                    )

                    __UpperCAmelCase    :						Dict       =						self.attna(
                        UpperCAmelCase_						,				encoder_hidden_states=UpperCAmelCase_						,				attention_mask=UpperCAmelCase_						,				**UpperCAmelCase_						,				)
                    __UpperCAmelCase    :						Optional[Any]       =						attn_output + hidden_states

             # 3. Feed-forward
             __UpperCAmelCase    :						str       =						self.norma(UpperCAmelCase_ )

             if self.use_ada_layer_norm_zero:
                    __UpperCAmelCase    :						List[str]       =						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

             if self._chunk_size is not None:
                    # "feed_forward_chunk_size" can be used to save memory
                    if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                           raise ValueError(
                               f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.''' )

                    __UpperCAmelCase    :						int       =						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
                    __UpperCAmelCase    :						List[Any]       =						torch.cat(
                        [self.ff(UpperCAmelCase_ ) for hid_slice in norm_hidden_states.chunk(UpperCAmelCase_						,				dim=self._chunk_dim )]						,				dim=self._chunk_dim						,				)
             else:
                    __UpperCAmelCase    :						Tuple       =						self.ff(UpperCAmelCase_ )

             if self.use_ada_layer_norm_zero:
                    __UpperCAmelCase    :						List[str]       =						gate_mlp.unsqueeze(1 ) * ff_output

             __UpperCAmelCase    :						List[str]       =						ff_output + hidden_states

             return hidden_states




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      Tuple						,				__lowercase       :      int						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      int = 4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "geglu"						,				__lowercase       :      bool = False						,				):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						int       =						int(dim * mult )
             __UpperCAmelCase    :						Any       =						dim_out if dim_out is not None else dim

             if activation_fn == "gelu":
                    __UpperCAmelCase    :						List[Any]       =						GELU(UpperCAmelCase_						,				UpperCAmelCase_ )
             if activation_fn == "gelu-approximate":
                    __UpperCAmelCase    :						Union[str, Any]       =						GELU(UpperCAmelCase_						,				UpperCAmelCase_						,				approximate='''tanh''' )
             elif activation_fn == "geglu":
                    __UpperCAmelCase    :						int       =						GEGLU(UpperCAmelCase_						,				UpperCAmelCase_ )
             elif activation_fn == "geglu-approximate":
                    __UpperCAmelCase    :						List[Any]       =						ApproximateGELU(UpperCAmelCase_						,				UpperCAmelCase_ )

             __UpperCAmelCase    :						Optional[Any]       =						nn.ModuleList([] )
             # project in
             self.net.append(UpperCAmelCase_ )
             # project dropout
             self.net.append(nn.Dropout(UpperCAmelCase_ ) )
             # project out
             self.net.append(nn.Linear(UpperCAmelCase_						,				UpperCAmelCase_ ) )
             # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
             if final_dropout:
                    self.net.append(nn.Dropout(UpperCAmelCase_ ) )







      def       A_					(							self       :      Optional[int]						,				__lowercase       :      int ):


             '''simple docstring'''




             for module in self.net:
                    __UpperCAmelCase    :						str       =						module(UpperCAmelCase_ )
             return hidden_states




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int						,				__lowercase       :      int						,				__lowercase       :      str = "none" ):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						int       =						nn.Linear(UpperCAmelCase_						,				UpperCAmelCase_ )
             __UpperCAmelCase    :						Any       =						approximate

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Any ):


             '''simple docstring'''




             if gate.device.type != "mps":
                    return F.gelu(UpperCAmelCase_						,				approximate=self.approximate )
             # mps: gelu is not implemented for float16
             return F.gelu(gate.to(dtype=torch.floataa )						,				approximate=self.approximate ).to(dtype=gate.dtype )







      def       A_					(							self       :      Dict						,				__lowercase       :      str ):


             '''simple docstring'''




             __UpperCAmelCase    :						Any       =						self.proj(UpperCAmelCase_ )
             __UpperCAmelCase    :						int       =						self.gelu(UpperCAmelCase_ )
             return hidden_states




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      List[Any]						,				__lowercase       :      int						,				__lowercase       :      int ):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						Tuple       =						nn.Linear(UpperCAmelCase_						,				dim_out * 2 )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Tuple ):


             '''simple docstring'''




             if gate.device.type != "mps":
                    return F.gelu(UpperCAmelCase_ )
             # mps: gelu is not implemented for float16
             return F.gelu(gate.to(dtype=torch.floataa ) ).to(dtype=gate.dtype )







      def       A_					(							self       :      List[Any]						,				__lowercase       :      int ):


             '''simple docstring'''




             __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						self.proj(UpperCAmelCase_ ).chunk(2						,				dim=-1 )
             return hidden_states * self.gelu(UpperCAmelCase_ )




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      Optional[int]						,				__lowercase       :      int						,				__lowercase       :      int ):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						int       =						nn.Linear(UpperCAmelCase_						,				UpperCAmelCase_ )







      def       A_					(							self       :      Any						,				__lowercase       :      Union[str, Any] ):


             '''simple docstring'''




             __UpperCAmelCase    :						str       =						self.proj(UpperCAmelCase_ )
             return x * torch.sigmoid(1.7_0_2 * x )




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      Optional[int]						,				__lowercase       :      List[Any]						,				__lowercase       :      int ):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						List[Any]       =						nn.Embedding(UpperCAmelCase_						,				UpperCAmelCase_ )
             __UpperCAmelCase    :						str       =						nn.SiLU()
             __UpperCAmelCase    :						Tuple       =						nn.Linear(UpperCAmelCase_						,				embedding_dim * 2 )
             __UpperCAmelCase    :						Optional[int]       =						nn.LayerNorm(UpperCAmelCase_						,				elementwise_affine=UpperCAmelCase_ )







      def       A_					(							self       :      Tuple						,				__lowercase       :      Optional[int]						,				__lowercase       :      Tuple ):


             '''simple docstring'''




             __UpperCAmelCase    :						Union[str, Any]       =						self.linear(self.silu(self.emb(UpperCAmelCase_ ) ) )
             __UpperCAmelCase       ,     __UpperCAmelCase    :						str       =						torch.chunk(UpperCAmelCase_						,				2 )
             __UpperCAmelCase    :						Any       =						self.norm(UpperCAmelCase_ ) * (1 + scale) + shift
             return x




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      List[str]						,				__lowercase       :      int						,				__lowercase       :      List[str] ):


             '''simple docstring'''




             super().__init__()

             __UpperCAmelCase    :						int       =						CombinedTimestepLabelEmbeddings(UpperCAmelCase_						,				UpperCAmelCase_ )

             __UpperCAmelCase    :						Tuple       =						nn.SiLU()
             __UpperCAmelCase    :						int       =						nn.Linear(UpperCAmelCase_						,				6 * embedding_dim						,				bias=UpperCAmelCase_ )
             __UpperCAmelCase    :						List[str]       =						nn.LayerNorm(UpperCAmelCase_						,				elementwise_affine=UpperCAmelCase_						,				eps=1e-6 )







      def       A_					(							self       :      List[str]						,				__lowercase       :      int						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Tuple						,				__lowercase       :      Any=None ):


             '''simple docstring'''




             __UpperCAmelCase    :						str       =						self.linear(self.silu(self.emb(UpperCAmelCase_						,				UpperCAmelCase_						,				hidden_dtype=UpperCAmelCase_ ) ) )
             __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Optional[Any]       =						emb.chunk(6						,				dim=1 )
             __UpperCAmelCase    :						int       =						self.norm(UpperCAmelCase_ ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
             return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      int						,				__lowercase       :      int						,				__lowercase       :      int						,				__lowercase       :      int						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      float = 1e-5 ):


             '''simple docstring'''




             super().__init__()
             __UpperCAmelCase    :						Optional[int]       =						num_groups
             __UpperCAmelCase    :						Any       =						eps

             if act_fn is None:
                    __UpperCAmelCase    :						Tuple       =						None
             else:
                    __UpperCAmelCase    :						List[str]       =						get_activation(UpperCAmelCase_ )

             __UpperCAmelCase    :						Any       =						nn.Linear(UpperCAmelCase_						,				out_dim * 2 )







      def       A_					(							self       :      List[str]						,				__lowercase       :      Any						,				__lowercase       :      List[str] ):


             '''simple docstring'''




             if self.act:
                    __UpperCAmelCase    :						Dict       =						self.act(UpperCAmelCase_ )
             __UpperCAmelCase    :						str       =						self.linear(UpperCAmelCase_ )
             __UpperCAmelCase    :						List[str]       =						emb[:, :, None, None]
             __UpperCAmelCase       ,     __UpperCAmelCase    :						List[str]       =						emb.chunk(2						,				dim=1 )

             __UpperCAmelCase    :						Optional[Any]       =						F.group_norm(UpperCAmelCase_						,				self.num_groups						,				eps=self.eps )
             __UpperCAmelCase    :						Any       =						x * (1 + scale) + shift
             return x






"""simple docstring"""





def       lowerCamelCase_       (   UpperCAmelCase_  )						->float:





  """simple docstring"""


  return 10 - x * x


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->float:





  """simple docstring"""


  if equation(UpperCAmelCase_  ) * equation(UpperCAmelCase_  ) >= 0:
    raise ValueError('''Wrong space!'''  )

  __UpperCAmelCase    :						Tuple       =						a
  while (b - a) >= 0.01:
    # Find middle point
    __UpperCAmelCase    :						List[str]       =						(a + b) / 2
    # Check if middle point is root
    if equation(UpperCAmelCase_  ) == 0.0:
      break
    # Decide the side to repeat the steps
    if equation(UpperCAmelCase_  ) * equation(UpperCAmelCase_  ) < 0:
      __UpperCAmelCase    :						Union[str, Any]       =						c
    else:
      __UpperCAmelCase    :						str       =						c
  return c


if __name__ == "__main__":
  import doctest

  doctest.testmod()

  print(bisection(-2, 5))
  print(bisection(0, 6))