


from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		List[str] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[Any]=0.0 ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		str = "geglu" ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = True ,   _UpperCAmelCase       :		str = "layer_norm" ,   _UpperCAmelCase       :		bool = False ,   )		->  Dict:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						only_cross_attention

							lowercase__	     =						(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
							lowercase__	     =						(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

							if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
									raise ValueError(
									    f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
									    f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''						)

							# Define 3 blocks. Each block has its own normalization layer.
							# 1. Self-Attn
							if self.use_ada_layer_norm:
									lowercase__	     =						AdaLayerNorm(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif self.use_ada_layer_norm_zero:
									lowercase__	     =						AdaLayerNormZero(_UpperCAmelCase ,   _UpperCAmelCase						)
							else:
									lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
							lowercase__	     =						Attention(
							    query_dim=_UpperCAmelCase ,   heads=_UpperCAmelCase ,   dim_head=_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   bias=_UpperCAmelCase ,   cross_attention_dim=cross_attention_dim if only_cross_attention else None ,   upcast_attention=_UpperCAmelCase ,   )

							# 2. Cross-Attn
							if cross_attention_dim is not None or double_self_attention:
									# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
									# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
									# the second cross attention block.
									lowercase__	     =						(
									    AdaLayerNorm(_UpperCAmelCase ,   _UpperCAmelCase						)
									    if self.use_ada_layer_norm
									    else nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
									)
									lowercase__	     =						Attention(
									    query_dim=_UpperCAmelCase ,   cross_attention_dim=cross_attention_dim if not double_self_attention else None ,   heads=_UpperCAmelCase ,   dim_head=_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   bias=_UpperCAmelCase ,   upcast_attention=_UpperCAmelCase ,   )  # is self-attn if encoder_hidden_states is none
							else:
									lowercase__	     =						None
									lowercase__	     =						None

							# 3. Feed-forward
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
							lowercase__	     =						FeedForward(_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   activation_fn=_UpperCAmelCase ,   final_dropout=_UpperCAmelCase						)

							# let chunk size default to None
							lowercase__	     =						None
							lowercase__	     =						0

					def     lowerCamelCase__	(self       :		List[Any] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		int						)		->  int:




							"""simple docstring"""


							lowercase__	     =						chunk_size
							lowercase__	     =						dim


					def     lowerCamelCase__	(self       :		Optional[int] ,   _UpperCAmelCase       :		torch.FloatTensor ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.LongTensor] = None ,   _UpperCAmelCase       :		Dict[str, Any] = None ,   _UpperCAmelCase       :		Optional[torch.LongTensor] = None ,   )		->  List[str]:




							"""simple docstring"""


							if self.use_ada_layer_norm:
									lowercase__	     =						self.norma(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif self.use_ada_layer_norm_zero:
									lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__	     =						self.norma(
									    _UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   hidden_dtype=hidden_states.dtype						)
							else:
									lowercase__	     =						self.norma(_UpperCAmelCase						)

							lowercase__	     =						cross_attention_kwargs if cross_attention_kwargs is not None else {}

							lowercase__	     =						self.attna(
							    _UpperCAmelCase ,   encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None ,   attention_mask=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							if self.use_ada_layer_norm_zero:
									lowercase__	     =						gate_msa.unsqueeze(1						) * attn_output
							lowercase__	     =						attn_output + hidden_states

							# 2. Cross-Attention
							if self.attna is not None:
									lowercase__	     =						(
									    self.norma(_UpperCAmelCase ,   _UpperCAmelCase						) if self.use_ada_layer_norm else self.norma(_UpperCAmelCase						)
									)

									lowercase__	     =						self.attna(
									    _UpperCAmelCase ,   encoder_hidden_states=_UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
									lowercase__	     =						attn_output + hidden_states

							# 3. Feed-forward
							lowercase__	     =						self.norma(_UpperCAmelCase						)

							if self.use_ada_layer_norm_zero:
									lowercase__	     =						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

							if self._chunk_size is not None:
									# "feed_forward_chunk_size" can be used to save memory
									if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
											raise ValueError(
											    f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''						)

									lowercase__	     =						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
									lowercase__	     =						torch.cat(
									    [self.ff(_UpperCAmelCase						) for hid_slice in norm_hidden_states.chunk(_UpperCAmelCase ,   dim=self._chunk_dim						)] ,   dim=self._chunk_dim ,   )
							else:
									lowercase__	     =						self.ff(_UpperCAmelCase						)

							if self.use_ada_layer_norm_zero:
									lowercase__	     =						gate_mlp.unsqueeze(1						) * ff_output

							lowercase__	     =						ff_output + hidden_states

							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		int = 4 ,   _UpperCAmelCase       :		float = 0.0 ,   _UpperCAmelCase       :		str = "geglu" ,   _UpperCAmelCase       :		bool = False ,   )		->  Union[str, Any]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						int(dim * mult						)
							lowercase__	     =						dim_out if dim_out is not None else dim

							if activation_fn == "gelu":
									lowercase__	     =						GELU(_UpperCAmelCase ,   _UpperCAmelCase						)
							if activation_fn == "gelu-approximate":
									lowercase__	     =						GELU(_UpperCAmelCase ,   _UpperCAmelCase ,   approximate="""tanh"""						)
							elif activation_fn == "geglu":
									lowercase__	     =						GEGLU(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif activation_fn == "geglu-approximate":
									lowercase__	     =						ApproximateGELU(_UpperCAmelCase ,   _UpperCAmelCase						)

							lowercase__	     =						nn.ModuleList([]						)
							# project in
							self.net.append(_UpperCAmelCase						)
							# project dropout
							self.net.append(nn.Dropout(_UpperCAmelCase						)						)
							# project out
							self.net.append(nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)						)
							# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
							if final_dropout:
									self.net.append(nn.Dropout(_UpperCAmelCase						)						)


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Optional[Any]						)		->  Tuple:




							"""simple docstring"""


							for module in self.net:
									lowercase__	     =						module(_UpperCAmelCase						)
							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		str = "none"						)		->  int:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)
							lowercase__	     =						approximate

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		List[str]						)		->  List[Any]:




							"""simple docstring"""


							if gate.device.type != "mps":
									return F.gelu(_UpperCAmelCase ,   approximate=self.approximate						)
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa						) ,   approximate=self.approximate						).to(dtype=gate.dtype						)


					def     lowerCamelCase__	(self       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str]						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.proj(_UpperCAmelCase						)
							lowercase__	     =						self.gelu(_UpperCAmelCase						)
							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  Union[str, Any]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   dim_out * 2						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Any						)		->  Optional[int]:




							"""simple docstring"""


							if gate.device.type != "mps":
									return F.gelu(_UpperCAmelCase						)
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa						)						).to(dtype=gate.dtype						)


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Optional[Any]						)		->  List[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.proj(_UpperCAmelCase						).chunk(2 ,   dim=-1						)
							return hidden_states * self.gelu(_UpperCAmelCase						)






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  str:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)


					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		int						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.proj(_UpperCAmelCase						)
							return x * torch.sigmoid(1.702 * x						)






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Union[str, Any]						)		->  str:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Embedding(_UpperCAmelCase ,   _UpperCAmelCase						)
							lowercase__	     =						nn.SiLU()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   embedding_dim * 2						)
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)


					def     lowerCamelCase__	(self       :		Tuple ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Dict						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.linear(self.silu(self.emb(_UpperCAmelCase						)						)						)
							lowercase__     ,      lowercase__	     =						torch.chunk(_UpperCAmelCase ,   2						)
							lowercase__	     =						self.norm(_UpperCAmelCase						) * (1 + scale) + shift
							return x






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Dict						)		->  List[str]:




							"""simple docstring"""


							super().__init__()

							lowercase__	     =						CombinedTimestepLabelEmbeddings(_UpperCAmelCase ,   _UpperCAmelCase						)

							lowercase__	     =						nn.SiLU()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   6 * embedding_dim ,   bias=_UpperCAmelCase						)
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase ,   eps=1E-6						)


					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int=None						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.linear(self.silu(self.emb(_UpperCAmelCase ,   _UpperCAmelCase ,   hidden_dtype=_UpperCAmelCase						)						)						)
							lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__	     =						emb.chunk(6 ,   dim=1						)
							lowercase__	     =						self.norm(_UpperCAmelCase						) * (1 + scale_msa[:, None]) + shift_msa[:, None]
							return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[str] = None ,   _UpperCAmelCase       :		float = 1E-5						)		->  Optional[int]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						num_groups
							lowercase__	     =						eps

							if act_fn is None:
									lowercase__	     =						None
							else:
									lowercase__	     =						get_activation(_UpperCAmelCase						)

							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   out_dim * 2						)


					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		List[Any]						)		->  int:




							"""simple docstring"""


							if self.act:
									lowercase__	     =						self.act(_UpperCAmelCase						)
							lowercase__	     =						self.linear(_UpperCAmelCase						)
							lowercase__	     =						emb[:, :, None, None]
							lowercase__     ,      lowercase__	     =						emb.chunk(2 ,   dim=1						)

							lowercase__	     =						F.group_norm(_UpperCAmelCase ,   self.num_groups ,   eps=self.eps						)
							lowercase__	     =						x * (1 + scale) + shift
							return x





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


A			:						Union[str, Any]        =  {
    'configuration_convnext': ['CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ConvNextConfig', 'ConvNextOnnxConfig']
}

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A			:						str        =  ['ConvNextFeatureExtractor']
	A			:						Optional[Any]        =  ['ConvNextImageProcessor']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A			:						str        =  [
	    'CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'ConvNextForImageClassification',
	    'ConvNextModel',
	    'ConvNextPreTrainedModel',
	    'ConvNextBackbone',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A			:						List[str]        =  [
	    'TFConvNextForImageClassification',
	    'TFConvNextModel',
	    'TFConvNextPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_convnext import CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvNextConfig, ConvNextOnnxConfig

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_convnext import ConvNextFeatureExtractor
		from .image_processing_convnext import ConvNextImageProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_convnext import (
		    CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    ConvNextBackbone,
		    ConvNextForImageClassification,
		    ConvNextModel,
		    ConvNextPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_convnext import TFConvNextForImageClassification, TFConvNextModel, TFConvNextPreTrainedModel


else:
	import sys

	A			:						int        =  _LazyModule(__name__, globals()['__file__'], _import_structure)

