


"""simple docstring"""


from typing import Callable, List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModel, CLIPTokenizer

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin, TransformeraDModel, VQModel
from ...schedulers import VQDiffusionScheduler
from ...utils import logging
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


A  :     Dict    					=    logging.get_logger(__name__)  # pylint: disable=invalid-name
class 						_UpperCamelCase						(     lowerCAmelCase__ ,lowerCAmelCase__	):
							'''simple docstring'''







							@register_to_config
							def __init__(  self					,     __a					,     __a = None					,     __a = None			):
										super().__init__()

										__lowerCAmelCase       					=    learnable

										if self.learnable:
													assert hidden_size is not None, "learnable=True requires `hidden_size` to be set"
													assert length is not None, "learnable=True requires `length` to be set"

													__lowerCAmelCase       					=    torch.zeros(__a					,     __a			)
										else:
													__lowerCAmelCase       					=    None

										__lowerCAmelCase       					=    torch.nn.Parameter(__a			)
class 						_UpperCamelCase						(     lowerCAmelCase__	):
							'''simple docstring'''







							__UpperCAmelCase		:					VQModel
							__UpperCAmelCase		:					CLIPTextModel
							__UpperCAmelCase		:					CLIPTokenizer
							__UpperCAmelCase		:					TransformeraDModel
							__UpperCAmelCase		:					LearnedClassifierFreeSamplingEmbeddings
							__UpperCAmelCase		:					VQDiffusionScheduler

							def __init__(  self					,     __a					,     __a					,     __a					,     __a					,     __a					,     __a					,     ):
										super().__init__()

										self.register_modules(
										    vqvae=__a					,     transformer=__a					,     text_encoder=__a					,     tokenizer=__a					,     scheduler=__a					,     learned_classifier_free_sampling_embeddings=__a					,     )

							def     snake_case       (  self					,     __a					,     __a					,     __a			):
										__lowerCAmelCase       					=    len(__a			) if isinstance(__a					,     __a			) else 1

										# get prompt text embeddings
										__lowerCAmelCase       					=    self.tokenizer(
										    __a					,     padding="max_length"					,     max_length=self.tokenizer.model_max_length					,     return_tensors="pt"					,     )
										__lowerCAmelCase       					=    text_inputs.input_ids

										if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
													__lowerCAmelCase       					=    self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]			)
													logger.warning(
													    "The following part of your input was truncated because CLIP can only handle sequences up to"
													    f" {self.tokenizer.model_max_length} tokens: {removed_text}"			)
													__lowerCAmelCase       					=    text_input_ids[:, : self.tokenizer.model_max_length]
										__lowerCAmelCase       					=    self.text_encoder(text_input_ids.to(self.device			)			)[0]

										# NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion.
										# While CLIP does normalize the pooled output of the text transformer when combining
										# the image and text embeddings, CLIP does not directly normalize the last hidden state.
										#
										# CLIP normalizing the pooled output.
										# https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053
										__lowerCAmelCase       					=    prompt_embeds / prompt_embeds.norm(dim=-1					,     keepdim=__a			)

										# duplicate text embeddings for each generation per prompt
										__lowerCAmelCase       					=    prompt_embeds.repeat_interleave(__a					,     dim=0			)

										if do_classifier_free_guidance:
													if self.learned_classifier_free_sampling_embeddings.learnable:
																__lowerCAmelCase       					=    self.learned_classifier_free_sampling_embeddings.embeddings
																__lowerCAmelCase       					=    negative_prompt_embeds.unsqueeze(0			).repeat(__a					,     1					,     1			)
													else:
																__lowerCAmelCase       					=    [""] * batch_size

																__lowerCAmelCase       					=    text_input_ids.shape[-1]
																__lowerCAmelCase       					=    self.tokenizer(
																    __a					,     padding="max_length"					,     max_length=__a					,     truncation=__a					,     return_tensors="pt"					,     )
																__lowerCAmelCase       					=    self.text_encoder(uncond_input.input_ids.to(self.device			)			)[0]
																# See comment for normalizing text embeddings
																__lowerCAmelCase       					=    negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1					,     keepdim=__a			)

													# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
													__lowerCAmelCase       					=    negative_prompt_embeds.shape[1]
													__lowerCAmelCase       					=    negative_prompt_embeds.repeat(1					,     __a					,     1			)
													__lowerCAmelCase       					=    negative_prompt_embeds.view(batch_size * num_images_per_prompt					,     __a					,     -1			)

													# For classifier free guidance, we need to do two forward passes.
													# Here we concatenate the unconditional and text embeddings into a single batch
													# to avoid doing two forward passes
													__lowerCAmelCase       					=    torch.cat([negative_prompt_embeds, prompt_embeds]			)

										return prompt_embeds

							@torch.no_grad()
							def __call__(  self					,     __a					,     __a = 1_00					,     __a = 5.0					,     __a = 1.0					,     __a = 1					,     __a = None					,     __a = None					,     __a = "pil"					,     __a = True					,     __a = None					,     __a = 1					,     ):
										if isinstance(__a					,     __a			):
													__lowerCAmelCase       					=    1
										elif isinstance(__a					,     __a			):
													__lowerCAmelCase       					=    len(__a			)
										else:
													raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(__a			)}"			)

										__lowerCAmelCase       					=    batch_size * num_images_per_prompt

										__lowerCAmelCase       					=    guidance_scale > 1.0

										__lowerCAmelCase       					=    self._encode_prompt(__a					,     __a					,     __a			)

										if (callback_steps is None) or (
										    callback_steps is not None and (not isinstance(__a					,     __a			) or callback_steps <= 0)
										):
													raise ValueError(
													    f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
													    f" {type(__a			)}."			)

										# get the initial completely masked latents unless the user supplied it

										__lowerCAmelCase       					=    (batch_size, self.transformer.num_latent_pixels)
										if latents is None:
													__lowerCAmelCase       					=    self.transformer.num_vector_embeds - 1
													__lowerCAmelCase       					=    torch.full(__a					,     __a			).to(self.device			)
										else:
													if latents.shape != latents_shape:
																raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}"			)
													if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any():
																raise ValueError(
																    "Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0,"
																    f" {self.transformer.num_vector_embeds - 1} (inclusive)."			)
													__lowerCAmelCase       					=    latents.to(self.device			)

										# set timesteps
										self.scheduler.set_timesteps(__a					,     device=self.device			)

										__lowerCAmelCase       					=    self.scheduler.timesteps.to(self.device			)

										__lowerCAmelCase       					=    latents

										for i, t in enumerate(self.progress_bar(__a			)			):
													# expand the sample if we are doing classifier free guidance
													__lowerCAmelCase       					=    torch.cat([sample] * 2			) if do_classifier_free_guidance else sample

													# predict the un-noised image
													# model_output == `log_p_x_0`
													__lowerCAmelCase       					=    self.transformer(__a					,     encoder_hidden_states=__a					,     timestep=__a			).sample

													if do_classifier_free_guidance:
																__lowerCAmelCase					,			__lowerCAmelCase       					=    model_output.chunk(2			)
																__lowerCAmelCase       					=    model_output_uncond + guidance_scale * (model_output_text - model_output_uncond)
																model_output -= torch.logsumexp(__a					,     dim=1					,     keepdim=__a			)

													__lowerCAmelCase       					=    self.truncate(__a					,     __a			)

													# remove `log(0)`'s (`-inf`s)
													__lowerCAmelCase       					=    model_output.clamp(-70			)

													# compute the previous noisy sample x_t -> x_t-1
													__lowerCAmelCase       					=    self.scheduler.step(__a					,     timestep=__a					,     sample=__a					,     generator=__a			).prev_sample

													# call the callback, if provided
													if callback is not None and i % callback_steps == 0:
																callback(__a					,     __a					,     __a			)

										__lowerCAmelCase       					=    self.vqvae.config.vq_embed_dim
										__lowerCAmelCase       					=    (batch_size, self.transformer.height, self.transformer.width, embedding_channels)
										__lowerCAmelCase       					=    self.vqvae.quantize.get_codebook_entry(__a					,     shape=__a			)
										__lowerCAmelCase       					=    self.vqvae.decode(__a					,     force_not_quantize=__a			).sample

										__lowerCAmelCase       					=    (image / 2 + 0.5).clamp(0					,     1			)
										__lowerCAmelCase       					=    image.cpu().permute(0					,     2					,     3					,     1			).numpy()

										if output_type == "pil":
													__lowerCAmelCase       					=    self.numpy_to_pil(__a			)

										if not return_dict:
													return (image,)

										return ImagePipelineOutput(images=__a			)



							def     snake_case       (  self					,     __a					,     __a			):
										__lowerCAmelCase					,			__lowerCAmelCase       					=    torch.sort(__a					,     1					,     descending=__a			)
										__lowerCAmelCase       					=    torch.exp(__a			)
										__lowerCAmelCase       					=    sorted_p_x_0.cumsum(dim=1			) < truncation_rate

										# Ensure that at least the largest probability is not zeroed out
										__lowerCAmelCase       					=    torch.full_like(keep_mask[:, 0:1, :]					,     __a			)
										__lowerCAmelCase       					=    torch.cat((all_true, keep_mask)					,     dim=1			)
										__lowerCAmelCase       					=    keep_mask[:, :-1, :]

										__lowerCAmelCase       					=    keep_mask.gather(1					,     indices.argsort(1			)			)

										__lowerCAmelCase       					=    log_p_x_0.clone()

										__lowerCAmelCase       					=    -torch.inf  # -inf = log(0)

										return rv








"""simple docstring"""


import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
	from sagemaker import Session, TrainingJobAnalytics
	from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv("""TEST_SAGEMAKER""" ,"""False"""	)	) is not True ,reason="""Skipping test because should only be run when releasing minor transformers version""" ,)
@pytest.mark.usefixtures("""sm_env"""	)
@parameterized_class(
    [
        {
            """framework""": """pytorch""",
            """script""": """run_glue_model_parallelism.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_6_0_0, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
        {
            """framework""": """pytorch""",
            """script""": """run_glue.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_6_0_0, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
    ]	)
class 						_UpperCamelCase						(     unittest.TestCase	):
							'''simple docstring'''







							def     snake_case       (  self			):
										if self.framework == "pytorch":
													subprocess.run(
													    f"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()					,     encoding="utf-8"					,     check=__a					,     )
										assert hasattr(self					,     "env"			)

							def     snake_case       (  self					,     __a			):
										# configuration for running training on smdistributed Model Parallel
										__lowerCAmelCase       					=    {
										    "enabled": True,
										    "processes_per_host": 8,
										}
										__lowerCAmelCase       					=    {
										    "enabled": True,
										    "parameters": {
										        "microbatches": 4,
										        "placement_strategy": "spread",
										        "pipeline": "interleaved",
										        "optimize": "speed",
										        "partitions": 4,
										        "ddp": True,
										    },
										}

										__lowerCAmelCase       					=    {"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

										__lowerCAmelCase       					=    "trainer" if self.script == "run_glue.py" else "smtrainer"
										# creates estimator
										return HuggingFace(
										    entry_point=self.script					,     source_dir=self.env.test_path					,     role=self.env.role					,     image_uri=self.env.image_uri					,     base_job_name=f"{self.env.base_job_name}-{instance_count}-smp-{name_extension}"					,     instance_count=__a					,     instance_type=self.instance_type					,     debugger_hook_config=__a					,     hyperparameters={
										        **self.env.hyperparameters,
										        "model_name_or_path": self.model_name_or_path,
										        "max_steps": 5_00,
										    }					,     metric_definitions=self.env.metric_definitions					,     distribution=__a					,     py_version="py36"					,     )

							def     snake_case       (  self					,     __a			):
										TrainingJobAnalytics(__a			).export_csv(f"{self.env.test_path}/{job_name}_metrics.csv"			)



							@parameterized.expand([(1,)]			)
							def     snake_case       (  self					,     __a			):
										# create estimator
										__lowerCAmelCase       					=    self.create_estimator(__a			)

										# run training
										estimator.fit()

										# result dataframe
										__lowerCAmelCase       					=    TrainingJobAnalytics(estimator.latest_training_job.name			).dataframe()

										# extract kpis
										__lowerCAmelCase       					=    list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]			)
										__lowerCAmelCase       					=    list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]			)
										# get train time from SageMaker job, this includes starting, preprocessing, stopping
										__lowerCAmelCase       					=    (
										    Session().describe_training_job(estimator.latest_training_job.name			).get("TrainingTimeInSeconds"					,     99_99_99			)
										)

										# assert kpis
										assert train_runtime <= self.results["train_runtime"]
										assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy			)
										assert all(t <= self.results["eval_loss"] for t in eval_loss			)

										# dump tests result into json file to share in PR
										with open(f"{estimator.latest_training_job.name}.json"					,     "w"			) as outfile:
													json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}					,     __a			)




