
from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .modeling_utils import ModelMixin
@dataclass
class 				__A					( A_      ):
			UpperCamelCase					:torch.FloatTensor







class 				__A					( A_					, A_      ):





			@register_to_config
			def __init__(self							,    __magic_name__ = 16							,    __magic_name__ = 88							,    __magic_name__ = None							,    __magic_name__ = None							,    __magic_name__ = 1							,    __magic_name__ = 0.0							,    __magic_name__ = 32							,    __magic_name__ = None							,    __magic_name__ = False							,    __magic_name__ = None							,    __magic_name__ = "geglu"							,    __magic_name__ = True							,    __magic_name__ = True							,    ):
						super().__init__()
						lowerCamelCase__ :      Union[str, Any]								=		num_attention_heads
						lowerCamelCase__ :      Optional[int]								=		attention_head_dim
						lowerCamelCase__ :      Optional[Any]								=		num_attention_heads * attention_head_dim

						lowerCamelCase__ :      Union[str, Any]								=		in_channels

						lowerCamelCase__ :      List[str]								=		torch.nn.GroupNorm(num_groups=__magic_name__							,    num_channels=__magic_name__							,    eps=1E-6							,    affine=__magic_name__				)
						lowerCamelCase__ :      Union[str, Any]								=		nn.Linear(__magic_name__							,    __magic_name__				)

						# 3. Define transformers blocks
						lowerCamelCase__ :      List[Any]								=		nn.ModuleList(
						    [
						        BasicTransformerBlock(
						            __magic_name__							,    __magic_name__							,    __magic_name__							,    dropout=__magic_name__							,    cross_attention_dim=__magic_name__							,    activation_fn=__magic_name__							,    attention_bias=__magic_name__							,    double_self_attention=__magic_name__							,    norm_elementwise_affine=__magic_name__							,    )
						        for d in range(__magic_name__				)
						    ]				)

						lowerCamelCase__ :      int								=		nn.Linear(__magic_name__							,    __magic_name__				)





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=1							,    __magic_name__=None							,    __magic_name__ = True							,    ):
						lowerCamelCase__ :      str								=		hidden_states.shape
						lowerCamelCase__ :      Union[str, Any]								=		batch_frames // num_frames

						lowerCamelCase__ :      List[str]								=		hidden_states

						lowerCamelCase__ :      Optional[int]								=		hidden_states[None, :].reshape(__magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__				)
						lowerCamelCase__ :      Optional[Any]								=		hidden_states.permute(0							,    2							,    1							,    3							,    4				)

						lowerCamelCase__ :      Optional[Any]								=		self.norm(__magic_name__				)
						lowerCamelCase__ :      str								=		hidden_states.permute(0							,    3							,    4							,    2							,    1				).reshape(batch_size * height * width							,    __magic_name__							,    __magic_name__				)

						lowerCamelCase__ :      str								=		self.proj_in(__magic_name__				)

						# 2. Blocks
						for block in self.transformer_blocks:
									lowerCamelCase__ :      str								=		block(
									    __magic_name__							,    encoder_hidden_states=__magic_name__							,    timestep=__magic_name__							,    cross_attention_kwargs=__magic_name__							,    class_labels=__magic_name__							,    )

						# 3. Output
						lowerCamelCase__ :      int								=		self.proj_out(__magic_name__				)
						lowerCamelCase__ :      Dict								=		(
						    hidden_states[None, None, :]
						    .reshape(__magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__				)
						    .permute(0							,    3							,    4							,    1							,    2				)
						    .contiguous()
						)
						lowerCamelCase__ :      int								=		hidden_states.reshape(__magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__				)

						lowerCamelCase__ :      Dict								=		hidden_states + residual

						if not return_dict:
									return (output,)

						return TransformerTemporalModelOutput(sample=__magic_name__				)








from ... import PretrainedConfig


_lowercase     							=						{
    '''sijunhe/nezha-cn-base''': '''https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json''',
}
class 				__A					( A_      ):
			UpperCamelCase					:Optional[Any]     =			NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
			UpperCamelCase					:int     =			'''nezha'''





			def __init__(self							,    __magic_name__=21128							,    __magic_name__=768							,    __magic_name__=12							,    __magic_name__=12							,    __magic_name__=3072							,    __magic_name__="gelu"							,    __magic_name__=0.1							,    __magic_name__=0.1							,    __magic_name__=512							,    __magic_name__=64							,    __magic_name__=2							,    __magic_name__=0.02							,    __magic_name__=1E-12							,    __magic_name__=0.1							,    __magic_name__=0							,    __magic_name__=2							,    __magic_name__=3							,    __magic_name__=True							,    **__magic_name__							,    ):
						super().__init__(pad_token_id=__magic_name__							,    bos_token_id=__magic_name__							,    eos_token_id=__magic_name__							,    **__magic_name__				)

						lowerCamelCase__ :      List[str]								=		vocab_size
						lowerCamelCase__ :      int								=		hidden_size
						lowerCamelCase__ :      int								=		num_hidden_layers
						lowerCamelCase__ :      Optional[int]								=		num_attention_heads
						lowerCamelCase__ :      Dict								=		hidden_act
						lowerCamelCase__ :      str								=		intermediate_size
						lowerCamelCase__ :      Optional[Any]								=		hidden_dropout_prob
						lowerCamelCase__ :      Dict								=		attention_probs_dropout_prob
						lowerCamelCase__ :      str								=		max_position_embeddings
						lowerCamelCase__ :      Optional[Any]								=		max_relative_position
						lowerCamelCase__ :      Optional[int]								=		type_vocab_size
						lowerCamelCase__ :      Optional[Any]								=		initializer_range
						lowerCamelCase__ :      Any								=		layer_norm_eps
						lowerCamelCase__ :      Union[str, Any]								=		classifier_dropout
						lowerCamelCase__ :      Union[str, Any]								=		use_cache



