




from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
@dataclass
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						lowerCAmelCase_							=  42







class 						UpperCAmelCase  (							UpperCAmelCase__			,  UpperCAmelCase__     ):







						'''simple docstring'''





						@register_to_config
						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	int = 32			,      __lowercase				:	int = 64			,      __lowercase				:	int = 20			,      __lowercase				:	int = 7_68			,      __lowercase				:	Dict=77			,      __lowercase				:	List[Any]=4			,      __lowercase				:	float = 0.0			,      __lowercase				:	str = "silu"			,      __lowercase				:	Optional[str] = None			,      __lowercase				:	Optional[str] = None			,      __lowercase				:	Optional[str] = "linear"			,      __lowercase				:	Optional[str] = "prd"			,      __lowercase				:	Optional[int] = None			,      __lowercase				:	Optional[int] = None			,      __lowercase				:	Optional[int] = None			,      ):






							"""simple docstring"""






							super().__init__()
							snake_case_						       =	num_attention_heads
							snake_case_						       =	attention_head_dim
							snake_case_						       =	num_attention_heads * attention_head_dim
							snake_case_						       =	additional_embeddings

							snake_case_						       =	time_embed_dim or inner_dim
							snake_case_						       =	embedding_proj_dim or embedding_dim
							snake_case_						       =	clip_embed_dim or embedding_dim

							snake_case_						       =	Timesteps(__lowercase			,      __lowercase			,      0		)
							snake_case_						       =	TimestepEmbedding(__lowercase			,      __lowercase			,      out_dim=__lowercase			,      act_fn=__lowercase		)

							snake_case_						       =	nn.Linear(__lowercase			,      __lowercase		)

							if embedding_proj_norm_type is None:
								snake_case_						       =	None
							elif embedding_proj_norm_type == "layer":
								snake_case_						       =	nn.LayerNorm(__lowercase		)
							else:
								raise ValueError(f"unsupported embedding_proj_norm_type: {embedding_proj_norm_type}"		)

							snake_case_						       =	nn.Linear(__lowercase			,      __lowercase		)

							if encoder_hid_proj_type is None:
								snake_case_						       =	None
							elif encoder_hid_proj_type == "linear":
								snake_case_						       =	nn.Linear(__lowercase			,      __lowercase		)
							else:
								raise ValueError(f"unsupported encoder_hid_proj_type: {encoder_hid_proj_type}"		)

							snake_case_						       =	nn.Parameter(torch.zeros(1			,      num_embeddings + additional_embeddings			,      __lowercase		)		)

							if added_emb_type == "prd":
								snake_case_						       =	nn.Parameter(torch.zeros(1			,      1			,      __lowercase		)		)
							elif added_emb_type is None:
								snake_case_						       =	None
							else:
								raise ValueError(
								    f"`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `'prd'` or `None`."		)

							snake_case_						       =	nn.ModuleList(
							    [
							        BasicTransformerBlock(
							            __lowercase			,      __lowercase			,      __lowercase			,      dropout=__lowercase			,      activation_fn="gelu"			,      attention_bias=__lowercase			,      )
							        for d in range(__lowercase		)
							    ]		)

							if norm_in_type == "layer":
								snake_case_						       =	nn.LayerNorm(__lowercase		)
							elif norm_in_type is None:
								snake_case_						       =	None
							else:
								raise ValueError(f"Unsupported norm_in_type: {norm_in_type}."		)

							snake_case_						       =	nn.LayerNorm(__lowercase		)

							snake_case_						       =	nn.Linear(__lowercase			,      __lowercase		)

							snake_case_						       =	torch.full(
							    [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]			,      -1_0000.0		)
							causal_attention_mask.triu_(1		)
							snake_case_						       =	causal_attention_mask[None, ...]
							self.register_buffer("causal_attention_mask"			,      __lowercase			,      persistent=__lowercase		)

							snake_case_						       =	nn.Parameter(torch.zeros(1			,      __lowercase		)		)
							snake_case_						       =	nn.Parameter(torch.zeros(1			,      __lowercase		)		)






						@property
						# Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
						def 			snake_case__		(		self				:	List[str]		):






							"""simple docstring"""






							snake_case_						       =	{}

							def fn_recursive_add_processors(__lowercase				:	str			,      __lowercase				:	torch.nn.Module			,      __lowercase				:	Dict[str, AttentionProcessor]		):
								if hasattr(__lowercase			,      "set_processor"		):
									snake_case_						       =	module.processor

								for sub_name, child in module.named_children():
									fn_recursive_add_processors(f"{name}.{sub_name}"			,      __lowercase			,      __lowercase		)

								return processors

							for name, module in self.named_children():
								fn_recursive_add_processors(__lowercase			,      __lowercase			,      __lowercase		)

							return processors






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Union[AttentionProcessor, Dict[str, AttentionProcessor]]		):






							"""simple docstring"""






							snake_case_						       =	len(self.attn_processors.keys()		)

							if isinstance(__lowercase			,      __lowercase		) and len(__lowercase		) != count:
								raise ValueError(
								    f"A dict of processors was passed, but the number of processors {len(__lowercase		)} does not match the"
								    f" number of attention layers: {count}. Please make sure to pass {count} processor classes."		)

							def fn_recursive_attn_processor(__lowercase				:	str			,      __lowercase				:	torch.nn.Module			,      __lowercase				:	List[Any]		):
								if hasattr(__lowercase			,      "set_processor"		):
									if not isinstance(__lowercase			,      __lowercase		):
										module.set_processor(__lowercase		)
									else:
										module.set_processor(processor.pop(f"{name}.processor"		)		)

								for sub_name, child in module.named_children():
									fn_recursive_attn_processor(f"{name}.{sub_name}"			,      __lowercase			,      __lowercase		)

							for name, module in self.named_children():
								fn_recursive_attn_processor(__lowercase			,      __lowercase			,      __lowercase		)






						def 			snake_case__		(		self				:	Any		):






							"""simple docstring"""






							self.set_attn_processor(AttnProcessor()		)






						def 			snake_case__		(		self				:	Dict			,      __lowercase				:	List[Any]			,      __lowercase				:	Union[torch.Tensor, float, int]			,      __lowercase				:	torch.FloatTensor			,      __lowercase				:	Optional[torch.FloatTensor] = None			,      __lowercase				:	Optional[torch.BoolTensor] = None			,      __lowercase				:	bool = True			,      ):






							"""simple docstring"""






							snake_case_						       =	hidden_states.shape[0]

							snake_case_						       =	timestep
							if not torch.is_tensor(__lowercase		):
								snake_case_						       =	torch.tensor([timesteps]			,      dtype=torch.long			,      device=hidden_states.device		)
							elif torch.is_tensor(__lowercase		) and len(timesteps.shape		) == 0:
								snake_case_						       =	timesteps[None].to(hidden_states.device		)

							# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
							snake_case_						       =	timesteps * torch.ones(__lowercase			,      dtype=timesteps.dtype			,      device=timesteps.device		)

							snake_case_						       =	self.time_proj(__lowercase		)

							# timesteps does not contain any weights and will always return f32 tensors
							# but time_embedding might be fp16, so we need to cast here.
							snake_case_						       =	timesteps_projected.to(dtype=self.dtype		)
							snake_case_						       =	self.time_embedding(__lowercase		)

							if self.embedding_proj_norm is not None:
								snake_case_						       =	self.embedding_proj_norm(__lowercase		)

							snake_case_						       =	self.embedding_proj(__lowercase		)
							if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
								snake_case_						       =	self.encoder_hidden_states_proj(__lowercase		)
							elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
								raise ValueError("`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set"		)

							snake_case_						       =	self.proj_in(__lowercase		)

							snake_case_						       =	self.positional_embedding.to(hidden_states.dtype		)

							snake_case_						       =	[]
							snake_case_						       =	0

							if encoder_hidden_states is not None:
								additional_embeds.append(__lowercase		)
								additional_embeddings_len += encoder_hidden_states.shape[1]

							if len(proj_embeddings.shape		) == 2:
								snake_case_						       =	proj_embeddings[:, None, :]

							if len(hidden_states.shape		) == 2:
								snake_case_						       =	hidden_states[:, None, :]

							snake_case_						       =	additional_embeds + [
							    proj_embeddings,
							    time_embeddings[:, None, :],
							    hidden_states,
							]

							if self.prd_embedding is not None:
								snake_case_						       =	self.prd_embedding.to(hidden_states.dtype		).expand(__lowercase			,      -1			,      -1		)
								additional_embeds.append(__lowercase		)

							snake_case_						       =	torch.cat(
							    __lowercase			,      dim=1			,      )

							# Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
							snake_case_						       =	additional_embeddings_len + proj_embeddings.shape[1] + 1
							if positional_embeddings.shape[1] < hidden_states.shape[1]:
								snake_case_						       =	F.pad(
								    __lowercase			,      (
								        0,
								        0,
								        additional_embeddings_len,
								        self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
								    )			,      value=0.0			,      )

							snake_case_						       =	hidden_states + positional_embeddings

							if attention_mask is not None:
								snake_case_						       =	(1 - attention_mask.to(hidden_states.dtype		)) * -1_0000.0
								snake_case_						       =	F.pad(__lowercase			,      (0, self.additional_embeddings)			,      value=0.0		)
								snake_case_						       =	(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype		)
								snake_case_						       =	attention_mask.repeat_interleave(self.config.num_attention_heads			,      dim=0		)

							if self.norm_in is not None:
								snake_case_						       =	self.norm_in(__lowercase		)

							for block in self.transformer_blocks:
								snake_case_						       =	block(__lowercase			,      attention_mask=__lowercase		)

							snake_case_						       =	self.norm_out(__lowercase		)

							if self.prd_embedding is not None:
								snake_case_						       =	hidden_states[:, -1]
							else:
								snake_case_						       =	hidden_states[:, additional_embeddings_len:]

							snake_case_						       =	self.proj_to_clip_embeddings(__lowercase		)

							if not return_dict:
								return (predicted_image_embedding,)

							return PriorTransformerOutput(predicted_image_embedding=__lowercase		)






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							snake_case_						       =	(prior_latents * self.clip_std) + self.clip_mean
							return prior_latents






import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
		from sagemaker import Session, TrainingJobAnalytics
		from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''			,  '''False'''     )     ) is not True			,  reason='''Skipping test because should only be run when releasing minor transformers version'''			,  )
@pytest.mark.usefixtures('''sm_env'''     )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 650, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_ddp.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf_dist.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.6, '''eval_loss''': 0.7},
        },
    ]     )
class 						UpperCAmelCase  (							unittest.TestCase     ):







						'''simple docstring'''





						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							if self.framework == "pytorch":
								subprocess.run(
								    f"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()			,      encoding="utf-8"			,      check=__lowercase			,      )
							assert hasattr(self			,      "env"		)






						def 			snake_case__		(		self				:	Dict			,      __lowercase				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	f"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
							# distributed data settings
							snake_case_						       =	{"smdistributed": {"dataparallel": {"enabled": True}}} if self.script != "run_ddp.py" else None

							# creates estimator
							return HuggingFace(
							    entry_point=self.script			,      source_dir=self.env.test_path			,      role=self.env.role			,      image_uri=self.env.image_uri			,      base_job_name=__lowercase			,      instance_count=__lowercase			,      instance_type=self.instance_type			,      debugger_hook_config=__lowercase			,      hyperparameters={**self.env.distributed_hyperparameters, "model_name_or_path": self.model_name_or_path}			,      metric_definitions=self.env.metric_definitions			,      distribution=__lowercase			,      py_version="py36"			,      )






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							TrainingJobAnalytics(__lowercase		).export_csv(f"{self.env.test_path}/{job_name}_metrics.csv"		)






						@parameterized.expand([(2,)]		)
						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							snake_case_						       =	self.create_estimator(__lowercase		)

							# run training
							estimator.fit()

							# result dataframe
							snake_case_						       =	TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

							# extract kpis
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]		)
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]		)
							# get train time from SageMaker job, this includes starting, preprocessing, stopping
							snake_case_						       =	(
							    Session().describe_training_job(estimator.latest_training_job.name		).get("TrainingTimeInSeconds"			,      99_99_99		)
							)

							# assert kpis
							assert train_runtime <= self.results["train_runtime"]
							assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy		)
							assert all(t <= self.results["eval_loss"] for t in eval_loss		)

							# dump tests result into json file to share in PR
							with open(f"{estimator.latest_training_job.name}.json"			,      "w"		) as outfile:
								json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}			,      __lowercase		)
