


from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						str					,			lowerCamelCase :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						Any=0.0					,			lowerCamelCase :						Optional[int] = None					,			lowerCamelCase :						str = "geglu"					,			lowerCamelCase :						Optional[int] = None					,			lowerCamelCase :						bool = False					,			lowerCamelCase :						bool = False					,			lowerCamelCase :						bool = False					,			lowerCamelCase :						bool = False					,			lowerCamelCase :						bool = True					,			lowerCamelCase :						str = "layer_norm"					,			lowerCamelCase :						bool = False					,			)			->					Optional[Any]:
														super().__init__()
														__snake_case	:    int											=     only_cross_attention

														__snake_case	:    Tuple											=     (num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
														__snake_case	:    Optional[Any]											=     (num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

														if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
																					raise ValueError(
																					    F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
																					    F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'       )

														# Define 3 blocks. Each block has its own normalization layer.
														# 1. Self-Attn
														if self.use_ada_layer_norm:
																					__snake_case	:    Union[str, Any]											=     AdaLayerNorm(UpperCAmelCase__					,			UpperCAmelCase__       )
														elif self.use_ada_layer_norm_zero:
																					__snake_case	:    List[Any]											=     AdaLayerNormZero(UpperCAmelCase__					,			UpperCAmelCase__       )
														else:
																					__snake_case	:    Tuple											=     nn.LayerNorm(UpperCAmelCase__					,			elementwise_affine=UpperCAmelCase__       )
														__snake_case	:    List[Any]											=     Attention(
														    query_dim=UpperCAmelCase__					,			heads=UpperCAmelCase__					,			dim_head=UpperCAmelCase__					,			dropout=UpperCAmelCase__					,			bias=UpperCAmelCase__					,			cross_attention_dim=cross_attention_dim if only_cross_attention else None					,			upcast_attention=UpperCAmelCase__					,			)

														# 2. Cross-Attn
														if cross_attention_dim is not None or double_self_attention:
																					# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
																					# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
																					# the second cross attention block.
																					__snake_case	:    List[Any]											=     (
																					    AdaLayerNorm(UpperCAmelCase__					,			UpperCAmelCase__       )
																					    if self.use_ada_layer_norm
																					    else nn.LayerNorm(UpperCAmelCase__					,			elementwise_affine=UpperCAmelCase__       )
																					)
																					__snake_case	:    Union[str, Any]											=     Attention(
																					    query_dim=UpperCAmelCase__					,			cross_attention_dim=cross_attention_dim if not double_self_attention else None					,			heads=UpperCAmelCase__					,			dim_head=UpperCAmelCase__					,			dropout=UpperCAmelCase__					,			bias=UpperCAmelCase__					,			upcast_attention=UpperCAmelCase__					,			)  # is self-attn if encoder_hidden_states is none
														else:
																					__snake_case	:    Union[str, Any]											=     None
																					__snake_case	:    Optional[Any]											=     None

														# 3. Feed-forward
														__snake_case	:    Dict											=     nn.LayerNorm(UpperCAmelCase__					,			elementwise_affine=UpperCAmelCase__       )
														__snake_case	:    str											=     FeedForward(UpperCAmelCase__					,			dropout=UpperCAmelCase__					,			activation_fn=UpperCAmelCase__					,			final_dropout=UpperCAmelCase__       )

														# let chunk size default to None
														__snake_case	:    List[str]											=     None
														__snake_case	:    List[str]											=     0

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						int       )			->					Union[str, Any]:
														__snake_case	:    Optional[Any]											=     chunk_size
														__snake_case	:    Optional[Any]											=     dim






							def 					__snake_case      (							self :						Dict					,			lowerCamelCase :						torch.FloatTensor					,			lowerCamelCase :						Optional[torch.FloatTensor] = None					,			lowerCamelCase :						Optional[torch.FloatTensor] = None					,			lowerCamelCase :						Optional[torch.FloatTensor] = None					,			lowerCamelCase :						Optional[torch.LongTensor] = None					,			lowerCamelCase :						Dict[str, Any] = None					,			lowerCamelCase :						Optional[torch.LongTensor] = None					,			)			->					List[Any]:
														# Notice that normalization is always applied before the real computation in the following blocks.
														# 1. Self-Attention
														if self.use_ada_layer_norm:
																					__snake_case	:    Dict											=     self.norma(UpperCAmelCase__					,			UpperCAmelCase__       )
														elif self.use_ada_layer_norm_zero:
																					__snake_case	:    str											=     self.norma(
																					    UpperCAmelCase__					,			UpperCAmelCase__					,			UpperCAmelCase__					,			hidden_dtype=hidden_states.dtype       )
														else:
																					__snake_case	:    Tuple											=     self.norma(UpperCAmelCase__       )

														__snake_case	:    List[str]											=     cross_attention_kwargs if cross_attention_kwargs is not None else {}

														__snake_case	:    Optional[Any]											=     self.attna(
														    UpperCAmelCase__					,			encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None					,			attention_mask=UpperCAmelCase__					,			**UpperCAmelCase__					,			)
														if self.use_ada_layer_norm_zero:
																					__snake_case	:    Optional[Any]											=     gate_msa.unsqueeze(1       ) * attn_output
														__snake_case	:    Any											=     attn_output + hidden_states

														# 2. Cross-Attention
														if self.attna is not None:
																					__snake_case	:    str											=     (
																					    self.norma(UpperCAmelCase__					,			UpperCAmelCase__       ) if self.use_ada_layer_norm else self.norma(UpperCAmelCase__       )
																					)

																					__snake_case	:    int											=     self.attna(
																					    UpperCAmelCase__					,			encoder_hidden_states=UpperCAmelCase__					,			attention_mask=UpperCAmelCase__					,			**UpperCAmelCase__					,			)
																					__snake_case	:    Tuple											=     attn_output + hidden_states

														# 3. Feed-forward
														__snake_case	:    List[str]											=     self.norma(UpperCAmelCase__       )

														if self.use_ada_layer_norm_zero:
																					__snake_case	:    Union[str, Any]											=     norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

														if self._chunk_size is not None:
																					# "feed_forward_chunk_size" can be used to save memory
																					if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																												raise ValueError(
																												    F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'       )

																					__snake_case	:    Any											=     norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
																					__snake_case	:    List[str]											=     torch.cat(
																					    [self.ff(UpperCAmelCase__       ) for hid_slice in norm_hidden_states.chunk(UpperCAmelCase__					,			dim=self._chunk_dim       )]					,			dim=self._chunk_dim					,			)
														else:
																					__snake_case	:    str											=     self.ff(UpperCAmelCase__       )

														if self.use_ada_layer_norm_zero:
																					__snake_case	:    int											=     gate_mlp.unsqueeze(1       ) * ff_output

														__snake_case	:    Any											=     ff_output + hidden_states

														return hidden_states

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						Optional[int]					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[int] = None					,			lowerCamelCase :						int = 4					,			lowerCamelCase :						float = 0.0					,			lowerCamelCase :						str = "geglu"					,			lowerCamelCase :						bool = False					,			)			->					Optional[int]:
														super().__init__()
														__snake_case	:    List[Any]											=     int(dim * mult       )
														__snake_case	:    Optional[int]											=     dim_out if dim_out is not None else dim

														if activation_fn == "gelu":
																					__snake_case	:    List[Any]											=     GELU(UpperCAmelCase__					,			UpperCAmelCase__       )
														if activation_fn == "gelu-approximate":
																					__snake_case	:    int											=     GELU(UpperCAmelCase__					,			UpperCAmelCase__					,			approximate="tanh"       )
														elif activation_fn == "geglu":
																					__snake_case	:    Optional[int]											=     GEGLU(UpperCAmelCase__					,			UpperCAmelCase__       )
														elif activation_fn == "geglu-approximate":
																					__snake_case	:    Tuple											=     ApproximateGELU(UpperCAmelCase__					,			UpperCAmelCase__       )

														__snake_case	:    Optional[Any]											=     nn.ModuleList([]       )
														# project in
														self.net.append(UpperCAmelCase__       )
														# project dropout
														self.net.append(nn.Dropout(UpperCAmelCase__       )       )
														# project out
														self.net.append(nn.Linear(UpperCAmelCase__					,			UpperCAmelCase__       )       )
														# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
														if final_dropout:
																					self.net.append(nn.Dropout(UpperCAmelCase__       )       )






							def 					__snake_case      (							self :						int					,			lowerCamelCase :						int       )			->					Optional[Any]:
														for module in self.net:
																					__snake_case	:    List[Any]											=     module(UpperCAmelCase__       )
														return hidden_states

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						List[str]					,			lowerCamelCase :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						str = "none"       )			->					Tuple:
														super().__init__()
														__snake_case	:    str											=     nn.Linear(UpperCAmelCase__					,			UpperCAmelCase__       )
														__snake_case	:    Union[str, Any]											=     approximate

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Optional[int]       )			->					str:
														if gate.device.type != "mps":
																					return F.gelu(UpperCAmelCase__					,			approximate=self.approximate       )
														# mps: gelu is not implemented for float16
														return F.gelu(gate.to(dtype=torch.floataa       )					,			approximate=self.approximate       ).to(dtype=gate.dtype       )






							def 					__snake_case      (							self :						List[Any]					,			lowerCamelCase :						Optional[int]       )			->					Optional[int]:
														__snake_case	:    Optional[int]											=     self.proj(UpperCAmelCase__       )
														__snake_case	:    Optional[Any]											=     self.gelu(UpperCAmelCase__       )
														return hidden_states

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						int       )			->					Tuple:
														super().__init__()
														__snake_case	:    List[Any]											=     nn.Linear(UpperCAmelCase__					,			dim_out * 2       )

							def 					__snake_case      (							self :						int					,			lowerCamelCase :						List[Any]       )			->					List[Any]:
														if gate.device.type != "mps":
																					return F.gelu(UpperCAmelCase__       )
														# mps: gelu is not implemented for float16
														return F.gelu(gate.to(dtype=torch.floataa       )       ).to(dtype=gate.dtype       )






							def 					__snake_case      (							self :						List[str]					,			lowerCamelCase :						List[str]       )			->					Tuple:
														__snake_case	:    Union[str, Any]											=     self.proj(UpperCAmelCase__       ).chunk(2					,			dim=-1       )
														return hidden_states * self.gelu(UpperCAmelCase__       )

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						Dict					,			lowerCamelCase :						int					,			lowerCamelCase :						int       )			->					List[str]:
														super().__init__()
														__snake_case	:    Tuple											=     nn.Linear(UpperCAmelCase__					,			UpperCAmelCase__       )






							def 					__snake_case      (							self :						Tuple					,			lowerCamelCase :						Tuple       )			->					str:
														__snake_case	:    Any											=     self.proj(UpperCAmelCase__       )
														return x * torch.sigmoid(1.7_02 * x       )

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						List[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Optional[int]       )			->					int:
														super().__init__()
														__snake_case	:    Tuple											=     nn.Embedding(UpperCAmelCase__					,			UpperCAmelCase__       )
														__snake_case	:    Dict											=     nn.SiLU()
														__snake_case	:    int											=     nn.Linear(UpperCAmelCase__					,			embedding_dim * 2       )
														__snake_case	:    List[Any]											=     nn.LayerNorm(UpperCAmelCase__					,			elementwise_affine=UpperCAmelCase__       )






							def 					__snake_case      (							self :						Any					,			lowerCamelCase :						int					,			lowerCamelCase :						Tuple       )			->					int:
														__snake_case	:    List[Any]											=     self.linear(self.silu(self.emb(UpperCAmelCase__       )       )       )
														__snake_case	:    Any											=     torch.chunk(UpperCAmelCase__					,			2       )
														__snake_case	:    Tuple											=     self.norm(UpperCAmelCase__       ) * (1 + scale) + shift
														return x

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						int					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						List[str]       )			->					List[str]:
														super().__init__()

														__snake_case	:    int											=     CombinedTimestepLabelEmbeddings(UpperCAmelCase__					,			UpperCAmelCase__       )

														__snake_case	:    Any											=     nn.SiLU()
														__snake_case	:    Optional[Any]											=     nn.Linear(UpperCAmelCase__					,			6 * embedding_dim					,			bias=UpperCAmelCase__       )
														__snake_case	:    Optional[int]											=     nn.LayerNorm(UpperCAmelCase__					,			elementwise_affine=UpperCAmelCase__					,			eps=1E-6       )






							def 					__snake_case      (							self :						Dict					,			lowerCamelCase :						Dict					,			lowerCamelCase :						str					,			lowerCamelCase :						Any					,			lowerCamelCase :						str=None       )			->					List[str]:
														__snake_case	:    Union[str, Any]											=     self.linear(self.silu(self.emb(UpperCAmelCase__					,			UpperCAmelCase__					,			hidden_dtype=UpperCAmelCase__       )       )       )
														__snake_case	:    int											=     emb.chunk(6					,			dim=1       )
														__snake_case	:    Dict											=     self.norm(UpperCAmelCase__       ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
														return x, gate_msa, shift_mlp, scale_mlp, gate_mlp

class        a	(nn.Module   ):




							"""simple docstring"""


							def __init__(							self :						Any					,			lowerCamelCase :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[str] = None					,			lowerCamelCase :						float = 1E-5       )			->					Union[str, Any]:
														super().__init__()
														__snake_case	:    Tuple											=     num_groups
														__snake_case	:    List[Any]											=     eps

														if act_fn is None:
																					__snake_case	:    Tuple											=     None
														else:
																					__snake_case	:    Optional[int]											=     get_activation(UpperCAmelCase__       )

														__snake_case	:    Optional[Any]											=     nn.Linear(UpperCAmelCase__					,			out_dim * 2       )






							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[Any]       )			->					int:
														if self.act:
																					__snake_case	:    List[Any]											=     self.act(UpperCAmelCase__       )
														__snake_case	:    Dict											=     self.linear(UpperCAmelCase__       )
														__snake_case	:    int											=     emb[:, :, None, None]
														__snake_case	:    List[Any]											=     emb.chunk(2					,			dim=1       )

														__snake_case	:    str											=     F.group_norm(UpperCAmelCase__					,			self.num_groups					,			eps=self.eps       )
														__snake_case	:    int											=     x * (1 + scale) + shift
														return x





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case  :	Optional[Any]			      =       logging.get_logger(__name__)

_snake_case  :	str			      =       {
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}

class        a	(_lowerCAmelCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							List[Any]  						=						"xlnet"
							__UpperCAmelCase					:							Dict  						=						["mems"]
							__UpperCAmelCase					:							Tuple  						=						{
							    "n_token": "vocab_size",  # Backward compatibility
							    "hidden_size": "d_model",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}

							def __init__(							self :						List[Any]					,			lowerCamelCase :						Optional[int]=32000					,			lowerCamelCase :						str=1024					,			lowerCamelCase :						int=24					,			lowerCamelCase :						List[Any]=16					,			lowerCamelCase :						Any=4096					,			lowerCamelCase :						Optional[Any]="gelu"					,			lowerCamelCase :						Optional[Any]=True					,			lowerCamelCase :						Optional[int]="bi"					,			lowerCamelCase :						Union[str, Any]=0.02					,			lowerCamelCase :						Union[str, Any]=1E-12					,			lowerCamelCase :						Dict=0.1					,			lowerCamelCase :						int=512					,			lowerCamelCase :						str=None					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						str=False					,			lowerCamelCase :						Any=False					,			lowerCamelCase :						Optional[Any]=-1					,			lowerCamelCase :						List[Any]=False					,			lowerCamelCase :						Optional[int]="last"					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						Union[str, Any]="tanh"					,			lowerCamelCase :						int=0.1					,			lowerCamelCase :						Any=5					,			lowerCamelCase :						List[str]=5					,			lowerCamelCase :						List[Any]=5					,			lowerCamelCase :						Tuple=1					,			lowerCamelCase :						Dict=2					,			**lowerCamelCase :						List[str]					,			)			->					List[Any]:
														__snake_case	:    Optional[Any]											=     vocab_size
														__snake_case	:    Any											=     d_model
														__snake_case	:    int											=     n_layer
														__snake_case	:    Optional[int]											=     n_head
														if d_model % n_head != 0:
																					raise ValueError(F'\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'       )
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    F'`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})'       )
														__snake_case	:    Any											=     d_model // n_head
														__snake_case	:    Tuple											=     ff_activation
														__snake_case	:    int											=     d_inner
														__snake_case	:    Optional[Any]											=     untie_r
														__snake_case	:    Optional[Any]											=     attn_type

														__snake_case	:    List[str]											=     initializer_range
														__snake_case	:    Tuple											=     layer_norm_eps

														__snake_case	:    str											=     dropout
														__snake_case	:    Any											=     mem_len
														__snake_case	:    Tuple											=     reuse_len
														__snake_case	:    Union[str, Any]											=     bi_data
														__snake_case	:    List[Any]											=     clamp_len
														__snake_case	:    Optional[int]											=     same_length

														__snake_case	:    List[str]											=     summary_type
														__snake_case	:    Any											=     summary_use_proj
														__snake_case	:    Any											=     summary_activation
														__snake_case	:    Optional[int]											=     summary_last_dropout
														__snake_case	:    Dict											=     start_n_top
														__snake_case	:    Optional[Any]											=     end_n_top

														__snake_case	:    List[str]											=     bos_token_id
														__snake_case	:    Any											=     pad_token_id
														__snake_case	:    Optional[Any]											=     eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead."					,			lowerCamelCase					,			)
																					__snake_case	:    Any											=     kwargs["use_cache"]

														__snake_case	:    int											=     use_mems_eval
														__snake_case	:    Optional[int]											=     use_mems_train
														super().__init__(pad_token_id=lowerCamelCase					,			bos_token_id=lowerCamelCase					,			eos_token_id=lowerCamelCase					,			**lowerCamelCase       )

							@property
							def 					__snake_case      (							self :						Any       )			->					Optional[int]:
														logger.info(F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )
														return -1






							@max_position_embeddings.setter
							def 					__snake_case      (							self :						Union[str, Any]					,			lowerCamelCase :						Any       )			->					Optional[Any]:
														# Message copied from Transformer-XL documentation
														raise NotImplementedError(
														    F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )

