

'''simple docstring'''

from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings


@maybe_allow_in_graph
class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Tuple    ,   __lowercase      :			int    ,   __lowercase      :			int    ,   __lowercase      :			int    ,   __lowercase      :			Optional[int]=0.0    ,   __lowercase      :			Optional[int] = None    ,   __lowercase      :			str = "geglu"    ,   __lowercase      :			Optional[int] = None    ,   __lowercase      :			bool = False    ,   __lowercase      :			bool = False    ,   __lowercase      :			bool = False    ,   __lowercase      :			bool = False    ,   __lowercase      :			bool = True    ,   __lowercase      :			str = "layer_norm"    ,   __lowercase      :			bool = False    ,   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=only_cross_attention

            __lowercase      		=(num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero'
            __lowercase      		=(num_embeds_ada_norm is not None) and norm_type == 'ada_norm'

            if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
                 raise ValueError(
                     f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
                     f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''   )

            # Define 3 blocks. Each block has its own normalization layer.
            # 1. Self-Attn
            if self.use_ada_layer_norm:
                 __lowercase      		=AdaLayerNorm(__lowercase    ,   __lowercase   )
            elif self.use_ada_layer_norm_zero:
                 __lowercase      		=AdaLayerNormZero(__lowercase    ,   __lowercase   )
            else:
                 __lowercase      		=nn.LayerNorm(__lowercase    ,   elementwise_affine=__lowercase   )
            __lowercase      		=Attention(
                query_dim=__lowercase    ,   heads=__lowercase    ,   dim_head=__lowercase    ,   dropout=__lowercase    ,   bias=__lowercase    ,   cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,   upcast_attention=__lowercase    ,   )

            # 2. Cross-Attn
            if cross_attention_dim is not None or double_self_attention:
                 # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
                 # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
                 # the second cross attention block.
                 __lowercase      		=(
                     AdaLayerNorm(__lowercase    ,   __lowercase   )
                     if self.use_ada_layer_norm
                     else nn.LayerNorm(__lowercase    ,   elementwise_affine=__lowercase   )
                 )
                 __lowercase      		=Attention(
                     query_dim=__lowercase    ,   cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,   heads=__lowercase    ,   dim_head=__lowercase    ,   dropout=__lowercase    ,   bias=__lowercase    ,   upcast_attention=__lowercase    ,   )  # is self-attn if encoder_hidden_states is none
            else:
                 __lowercase      		=None
                 __lowercase      		=None

            # 3. Feed-forward
            __lowercase      		=nn.LayerNorm(__lowercase    ,   elementwise_affine=__lowercase   )
            __lowercase      		=FeedForward(__lowercase    ,   dropout=__lowercase    ,   activation_fn=__lowercase    ,   final_dropout=__lowercase   )

            # let chunk size default to None
            __lowercase      		=None
            __lowercase      		=0







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			Optional[int]    ,   __lowercase      :			int   ):







            """simple docstring"""







            __lowercase      		=chunk_size
            __lowercase      		=dim







       def       snake_case (				self      :			Optional[int]    ,   __lowercase      :			torch.FloatTensor    ,   __lowercase      :			Optional[torch.FloatTensor] = None    ,   __lowercase      :			Optional[torch.FloatTensor] = None    ,   __lowercase      :			Optional[torch.FloatTensor] = None    ,   __lowercase      :			Optional[torch.LongTensor] = None    ,   __lowercase      :			Dict[str, Any] = None    ,   __lowercase      :			Optional[torch.LongTensor] = None    ,   ):







            """simple docstring"""







            if self.use_ada_layer_norm:
                 __lowercase      		=self.norma(__lowercase    ,   __lowercase   )
            elif self.use_ada_layer_norm_zero:
                 __lowercase		,  __lowercase		,  __lowercase		,  __lowercase		,  __lowercase      		=self.norma(
                     __lowercase    ,   __lowercase    ,   __lowercase    ,   hidden_dtype=hidden_states.dtype   )
            else:
                 __lowercase      		=self.norma(__lowercase   )

            __lowercase      		=cross_attention_kwargs if cross_attention_kwargs is not None else {}

            __lowercase      		=self.attna(
                __lowercase    ,   encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,   attention_mask=__lowercase    ,   **__lowercase    ,   )
            if self.use_ada_layer_norm_zero:
                 __lowercase      		=gate_msa.unsqueeze(1   ) * attn_output
            __lowercase      		=attn_output + hidden_states

            # 2. Cross-Attention
            if self.attna is not None:
                 __lowercase      		=(
                     self.norma(__lowercase    ,   __lowercase   ) if self.use_ada_layer_norm else self.norma(__lowercase   )
                 )

                 __lowercase      		=self.attna(
                     __lowercase    ,   encoder_hidden_states=__lowercase    ,   attention_mask=__lowercase    ,   **__lowercase    ,   )
                 __lowercase      		=attn_output + hidden_states

            # 3. Feed-forward
            __lowercase      		=self.norma(__lowercase   )

            if self.use_ada_layer_norm_zero:
                 __lowercase      		=norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

            if self._chunk_size is not None:
                 # "feed_forward_chunk_size" can be used to save memory
                 if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                      raise ValueError(
                          f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''   )

                 __lowercase      		=norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
                 __lowercase      		=torch.cat(
                     [self.ff(__lowercase   ) for hid_slice in norm_hidden_states.chunk(__lowercase    ,   dim=self._chunk_dim   )]    ,   dim=self._chunk_dim    ,   )
            else:
                 __lowercase      		=self.ff(__lowercase   )

            if self.use_ada_layer_norm_zero:
                 __lowercase      		=gate_mlp.unsqueeze(1   ) * ff_output

            __lowercase      		=ff_output + hidden_states

            return hidden_states






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Tuple    ,   __lowercase      :			int    ,   __lowercase      :			Optional[int] = None    ,   __lowercase      :			int = 4    ,   __lowercase      :			float = 0.0    ,   __lowercase      :			str = "geglu"    ,   __lowercase      :			bool = False    ,   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=int(dim * mult   )
            __lowercase      		=dim_out if dim_out is not None else dim

            if activation_fn == "gelu":
                 __lowercase      		=GELU(__lowercase    ,   __lowercase   )
            if activation_fn == "gelu-approximate":
                 __lowercase      		=GELU(__lowercase    ,   __lowercase    ,   approximate='tanh'   )
            elif activation_fn == "geglu":
                 __lowercase      		=GEGLU(__lowercase    ,   __lowercase   )
            elif activation_fn == "geglu-approximate":
                 __lowercase      		=ApproximateGELU(__lowercase    ,   __lowercase   )

            __lowercase      		=nn.ModuleList([]   )
            # project in
            self.net.append(__lowercase   )
            # project dropout
            self.net.append(nn.Dropout(__lowercase   )   )
            # project out
            self.net.append(nn.Linear(__lowercase    ,   __lowercase   )   )
            # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
            if final_dropout:
                 self.net.append(nn.Dropout(__lowercase   )   )







       def       snake_case (				self      :			str    ,   __lowercase      :			Optional[int]   ):







            """simple docstring"""







            for module in self.net:
                 __lowercase      		=module(__lowercase   )
            return hidden_states






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			int    ,   __lowercase      :			int    ,   __lowercase      :			int    ,   __lowercase      :			str = "none"   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=nn.Linear(__lowercase    ,   __lowercase   )
            __lowercase      		=approximate







       def       snake_case (				self      :			Dict    ,   __lowercase      :			List[Any]   ):







            """simple docstring"""







            if gate.device.type != "mps":
                 return F.gelu(__lowercase    ,   approximate=self.approximate   )
            # mps: gelu is not implemented for float16
            return F.gelu(gate.to(dtype=torch.floataa   )    ,   approximate=self.approximate   ).to(dtype=gate.dtype   )







       def       snake_case (				self      :			Any    ,   __lowercase      :			Tuple   ):







            """simple docstring"""







            __lowercase      		=self.proj(__lowercase   )
            __lowercase      		=self.gelu(__lowercase   )
            return hidden_states






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Tuple    ,   __lowercase      :			int    ,   __lowercase      :			int   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=nn.Linear(__lowercase    ,   dim_out * 2   )







       def       snake_case (				self      :			Tuple    ,   __lowercase      :			List[str]   ):







            """simple docstring"""







            if gate.device.type != "mps":
                 return F.gelu(__lowercase   )
            # mps: gelu is not implemented for float16
            return F.gelu(gate.to(dtype=torch.floataa   )   ).to(dtype=gate.dtype   )







       def       snake_case (				self      :			str    ,   __lowercase      :			int   ):







            """simple docstring"""







            __lowercase		,  __lowercase      		=self.proj(__lowercase   ).chunk(2    ,   dim=-1   )
            return hidden_states * self.gelu(__lowercase   )






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Optional[Any]    ,   __lowercase      :			int    ,   __lowercase      :			int   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=nn.Linear(__lowercase    ,   __lowercase   )







       def       snake_case (				self      :			List[Any]    ,   __lowercase      :			List[str]   ):







            """simple docstring"""







            __lowercase      		=self.proj(__lowercase   )
            return x * torch.sigmoid(1.7_0_2 * x   )






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			List[Any]    ,   __lowercase      :			Optional[Any]    ,   __lowercase      :			Any   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=nn.Embedding(__lowercase    ,   __lowercase   )
            __lowercase      		=nn.SiLU()
            __lowercase      		=nn.Linear(__lowercase    ,   embedding_dim * 2   )
            __lowercase      		=nn.LayerNorm(__lowercase    ,   elementwise_affine=__lowercase   )







       def       snake_case (				self      :			Union[str, Any]    ,   __lowercase      :			List[Any]    ,   __lowercase      :			Optional[int]   ):







            """simple docstring"""







            __lowercase      		=self.linear(self.silu(self.emb(__lowercase   )   )   )
            __lowercase		,  __lowercase      		=torch.chunk(__lowercase    ,   2   )
            __lowercase      		=self.norm(__lowercase   ) * (1 + scale) + shift
            return x






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Optional[int]    ,   __lowercase      :			Optional[int]    ,   __lowercase      :			str   ):







            """simple docstring"""







            super().__init__()

            __lowercase      		=CombinedTimestepLabelEmbeddings(__lowercase    ,   __lowercase   )

            __lowercase      		=nn.SiLU()
            __lowercase      		=nn.Linear(__lowercase    ,   6 * embedding_dim    ,   bias=__lowercase   )
            __lowercase      		=nn.LayerNorm(__lowercase    ,   elementwise_affine=__lowercase    ,   eps=1E-6   )







       def       snake_case (				self      :			Dict    ,   __lowercase      :			List[Any]    ,   __lowercase      :			int    ,   __lowercase      :			Tuple    ,   __lowercase      :			int=None   ):







            """simple docstring"""







            __lowercase      		=self.linear(self.silu(self.emb(__lowercase    ,   __lowercase    ,   hidden_dtype=__lowercase   )   )   )
            __lowercase		,  __lowercase		,  __lowercase		,  __lowercase		,  __lowercase		,  __lowercase      		=emb.chunk(6    ,   dim=1   )
            __lowercase      		=self.norm(__lowercase   ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
            return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class 	lowerCAmelCase  (   nn.Module ):







       def __init__(				self      :			Optional[Any]    ,   __lowercase      :			int    ,   __lowercase      :			int    ,   __lowercase      :			int    ,   __lowercase      :			Optional[str] = None    ,   __lowercase      :			float = 1E-5   ):







            """simple docstring"""







            super().__init__()
            __lowercase      		=num_groups
            __lowercase      		=eps

            if act_fn is None:
                 __lowercase      		=None
            else:
                 __lowercase      		=get_activation(__lowercase   )

            __lowercase      		=nn.Linear(__lowercase    ,   out_dim * 2   )







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			Tuple    ,   __lowercase      :			Dict   ):







            """simple docstring"""







            if self.act:
                 __lowercase      		=self.act(__lowercase   )
            __lowercase      		=self.linear(__lowercase   )
            __lowercase      		=emb[:, :, None, None]
            __lowercase		,  __lowercase      		=emb.chunk(2    ,   dim=1   )

            __lowercase      		=F.group_norm(__lowercase    ,   self.num_groups    ,   eps=self.eps   )
            __lowercase      		=x * (1 + scale) + shift
            return x





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase					 =	{
    '''configuration_owlvit''': [
        '''OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''OwlViTConfig''',
        '''OwlViTOnnxConfig''',
        '''OwlViTTextConfig''',
        '''OwlViTVisionConfig''',
    ],
    '''processing_owlvit''': ['''OwlViTProcessor'''],
}


try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase					 =	['''OwlViTFeatureExtractor''']
       UpperCAmelCase					 =	['''OwlViTImageProcessor''']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase					 =	[
           '''OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''OwlViTModel''',
           '''OwlViTPreTrainedModel''',
           '''OwlViTTextModel''',
           '''OwlViTVisionModel''',
           '''OwlViTForObjectDetection''',
       ]

if TYPE_CHECKING:
       from .configuration_owlvit import (
           OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
           OwlViTConfig,
           OwlViTOnnxConfig,
           OwlViTTextConfig,
           OwlViTVisionConfig,
       )
       from .processing_owlvit import OwlViTProcessor

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_owlvit import OwlViTFeatureExtractor
              from .image_processing_owlvit import OwlViTImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_owlvit import (
                  OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  OwlViTForObjectDetection,
                  OwlViTModel,
                  OwlViTPreTrainedModel,
                  OwlViTTextModel,
                  OwlViTVisionModel,
              )

else:
       import sys

       UpperCAmelCase					 =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


