from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .modeling_utils import ModelMixin


@dataclass
class       UpperCAmelCase__     (		__snake_case       ):



  """simple docstring"""






  lowerCAmelCase__		:						torch.FloatTensor






class       UpperCAmelCase__     (		__snake_case		,						__snake_case       ):



  """simple docstring"""






  @register_to_config
  def __init__(					self:      Optional[int]					,					__lowerCAmelCase:      Optional[int] = 16					,					__lowerCAmelCase:      int = 88					,					__lowerCAmelCase:      int = None					,					__lowerCAmelCase:      int = None					,					__lowerCAmelCase:      Union[str, Any] = 1					,					__lowerCAmelCase:      Optional[Any] = 0.0					,					__lowerCAmelCase:      List[str] = 32					,					__lowerCAmelCase:      Dict = None					,					__lowerCAmelCase:      List[Any] = False					,					__lowerCAmelCase:      List[Any] = None					,					__lowerCAmelCase:      int = "geglu"					,					__lowerCAmelCase:      List[str] = True					,					__lowerCAmelCase:      str = True					,					)			->			Tuple:




    '''simple docstring'''




    super().__init__()
    __UpperCAmelCase         =							num_attention_heads
    __UpperCAmelCase         =							attention_head_dim
    __UpperCAmelCase         =							num_attention_heads * attention_head_dim

    __UpperCAmelCase         =							in_channels

    __UpperCAmelCase         =							torch.nn.GroupNorm(num_groups=A__					,					num_channels=A__					,					eps=1E-6					,					affine=A__ )
    __UpperCAmelCase         =							nn.Linear(A__					,					A__ )

    # 3. Define transformers blocks
    __UpperCAmelCase         =							nn.ModuleList(
        [
            BasicTransformerBlock(
                A__					,					A__					,					A__					,					dropout=A__					,					cross_attention_dim=A__					,					activation_fn=A__					,					attention_bias=A__					,					double_self_attention=A__					,					norm_elementwise_affine=A__					,					)
            for d in range(A__ )
        ] )

    __UpperCAmelCase         =							nn.Linear(A__					,					A__ )






  def 						_UpperCAmelCase					(					self:      Any					,					__lowerCAmelCase:      int					,					__lowerCAmelCase:      List[Any]=None					,					__lowerCAmelCase:      int=None					,					__lowerCAmelCase:      Dict=None					,					__lowerCAmelCase:      List[str]=1					,					__lowerCAmelCase:      Any=None					,					__lowerCAmelCase:      str = True					,					)			->			Union[str, Any]:




    '''simple docstring'''




    __UpperCAmelCase         =							hidden_states.shape
    __UpperCAmelCase         =							batch_frames // num_frames

    __UpperCAmelCase         =							hidden_states

    __UpperCAmelCase         =							hidden_states[None, :].reshape(A__					,					A__					,					A__					,					A__					,					A__ )
    __UpperCAmelCase         =							hidden_states.permute(0					,					2					,					1					,					3					,					4 )

    __UpperCAmelCase         =							self.norm(A__ )
    __UpperCAmelCase         =							hidden_states.permute(0					,					3					,					4					,					2					,					1 ).reshape(batch_size * height * width					,					A__					,					A__ )

    __UpperCAmelCase         =							self.proj_in(A__ )

    # 2. Blocks
    for block in self.transformer_blocks:
      __UpperCAmelCase         =							block(
          A__					,					encoder_hidden_states=A__					,					timestep=A__					,					cross_attention_kwargs=A__					,					class_labels=A__					,					)

    # 3. Output
    __UpperCAmelCase         =							self.proj_out(A__ )
    __UpperCAmelCase         =							(
        hidden_states[None, None, :]
        .reshape(A__					,					A__					,					A__					,					A__					,					A__ )
        .permute(0					,					3					,					4					,					1					,					2 )
        .contiguous()
    )
    __UpperCAmelCase         =							hidden_states.reshape(A__					,					A__					,					A__					,					A__ )

    __UpperCAmelCase         =							hidden_states + residual

    if not return_dict:
      return (output,)

    return TransformerTemporalModelOutput(sample=A__ )











import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


_UpperCamelCase			:	Optional[Any]								=logging.get_logger(__name__)



def  a__ ()		-> Any:
				# Get the sagemaker specific mp parameters from smp_options variable.
				_A	:				Any   =      os.getenv('''SM_HP_MP_PARAMETERS'''					,  '''{}'''   )
				try:
								# Parse it and check the field "partitions" is included, it is required for model parallel.
								_A	:				Dict   =      json.loads(__lowercase   )
								if "partitions" not in smp_options:
												return False
				except json.JSONDecodeError:
								return False

				# Get the sagemaker specific framework parameters from mpi_options variable.
				_A	:				Optional[int]   =      os.getenv('''SM_FRAMEWORK_PARAMS'''					,  '''{}'''   )
				try:
								# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
								_A	:				Any   =      json.loads(__lowercase   )
								if not mpi_options.get('''sagemaker_mpi_enabled'''					,  __lowercase   ):
												return False
				except json.JSONDecodeError:
								return False
				# Lastly, check if the `smdistributed` module is present.
				return importlib.util.find_spec('''smdistributed'''   ) is not None


if is_sagemaker_model_parallel_available():
	import smdistributed.modelparallel.torch as smp

	smp.init()






@dataclass
class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       str     =   field(
		    default=""     ,				metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}     ,				)







		def  A__      (					self		):
						super().__post_init__()
						warnings.warn(
						    '''`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '''
						    '''`TrainingArguments` instead.'''       ,A__       ,)







		@cached_property
		def  A__      (					self		):
						logger.info('''PyTorch: setting up devices'''		)
						if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
										logger.warning(
										    '''torch.distributed process group is initialized, but local_rank == -1. '''
										    '''In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'''		)
						if self.no_cuda:
										_A	:				Union[str, Any]   =      torch.device('''cpu'''		)
										_A	:				List[str]   =      0
						elif is_sagemaker_model_parallel_available():
										_A	:				Dict   =      smp.local_rank()
										_A	:				Union[str, Any]   =      torch.device('''cuda'''       ,A__		)
										_A	:				Tuple   =      1
						elif is_sagemaker_dp_enabled():
										import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

										torch.distributed.init_process_group(backend='''smddp'''       ,timeout=self.ddp_timeout_delta		)
										_A	:				int   =      int(os.getenv('''SMDATAPARALLEL_LOCAL_RANK'''		)		)
										_A	:				int   =      torch.device('''cuda'''       ,self.local_rank		)
										_A	:				Union[str, Any]   =      1
						elif self.local_rank == -1:
										# if n_gpu is > 1 we'll use nn.DataParallel.
										# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
										# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
										# trigger an error that a device index is missing. Index 0 takes into account the
										# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
										# will use the first GPU in that env, i.e. GPU#1
										_A	:				Dict   =      torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''		)
										# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
										# the default value.
										_A	:				Union[str, Any]   =      torch.cuda.device_count()
						else:
										# Here, we'll use torch.distributed.
										# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
										if not torch.distributed.is_initialized():
														torch.distributed.init_process_group(backend='''nccl'''       ,timeout=self.ddp_timeout_delta		)
										_A	:				Union[str, Any]   =      torch.device('''cuda'''       ,self.local_rank		)
										_A	:				List[Any]   =      1

						if device.type == "cuda":
										torch.cuda.set_device(A__		)

						return device







		@property
		def  A__      (					self		):
						if is_sagemaker_model_parallel_available():
										return smp.dp_size()

						return super().world_size







		@property
		def  A__      (					self		):
						return not is_sagemaker_model_parallel_available()







		@property
		def  A__      (					self		):
						return False



