





import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

import diffusers
from diffusers import (
    AutoencoderKL,
    EulerDiscreteScheduler,
    StableDiffusionLatentUpscalePipeline,
    StableDiffusionPipeline,
    UNetaDConditionModel,
)
from diffusers.schedulers import KarrasDiffusionSchedulers
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()
def      A__       (    SCREAMING_SNAKE_CASE_   :      List[Any]						)  ->  Any:






   """simple docstring"""







   _UpperCAmelCase           =   [tensor.shape for tensor in tensor_list]
   return all(shape == shapes[0] for shape in shapes[1:]						)




class 			__UpperCamelCase    (							A__	, A__	, A__	, unittest.TestCase    ):
     __A	:  str							  =   StableDiffusionLatentUpscalePipeline
     __A	:  int							  =   TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {
         """height""",
         """width""",
         """cross_attention_kwargs""",
         """negative_prompt_embeds""",
         """prompt_embeds""",
     }
     __A	:  Dict							  =   PipelineTesterMixin.required_optional_params - {"""num_images_per_prompt"""}
     __A	:  Dict							  =   TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
     __A	:  Any							  =   frozenset(
         []    )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
     __A	:  Optional[int]							  =   frozenset([]    )

     __A	:  str							  =   True





     @property
     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   1
        _UpperCAmelCase           =   4
        _UpperCAmelCase           =   (16, 16)

        _UpperCAmelCase           =   floats_tensor((batch_size, num_channels) + sizes				,							rng=random.Random(0					)					).to(_UpperCamelCase					)
        return image





     def 		UpperCamelCase(     self					):
        torch.manual_seed(0					)
        _UpperCAmelCase           =   UNetaDConditionModel(
            act_fn='''gelu'''				,							attention_head_dim=8				,							norm_num_groups=_UpperCamelCase				,							block_out_channels=[32, 32, 64, 64]				,							time_cond_proj_dim=160				,							conv_in_kernel=1				,							conv_out_kernel=1				,							cross_attention_dim=32				,							down_block_types=(
                '''KDownBlock2D''',
                '''KCrossAttnDownBlock2D''',
                '''KCrossAttnDownBlock2D''',
                '''KCrossAttnDownBlock2D''',
            )				,							in_channels=8				,							mid_block_type=_UpperCamelCase				,							only_cross_attention=_UpperCamelCase				,							out_channels=5				,							resnet_time_scale_shift='''scale_shift'''				,							time_embedding_type='''fourier'''				,							timestep_post_act='''gelu'''				,							up_block_types=('''KCrossAttnUpBlock2D''', '''KCrossAttnUpBlock2D''', '''KCrossAttnUpBlock2D''', '''KUpBlock2D''')				,							)
        _UpperCAmelCase           =   AutoencoderKL(
            block_out_channels=[32, 32, 64, 64]				,							in_channels=3				,							out_channels=3				,							down_block_types=[
                '''DownEncoderBlock2D''',
                '''DownEncoderBlock2D''',
                '''DownEncoderBlock2D''',
                '''DownEncoderBlock2D''',
            ]				,							up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''', '''UpDecoderBlock2D''', '''UpDecoderBlock2D''']				,							latent_channels=4				,							)
        _UpperCAmelCase           =   EulerDiscreteScheduler(prediction_type='''sample'''					)
        _UpperCAmelCase           =   CLIPTextConfig(
            bos_token_id=0				,							eos_token_id=2				,							hidden_size=32				,							intermediate_size=37				,							layer_norm_eps=1e-05				,							num_attention_heads=4				,							num_hidden_layers=5				,							pad_token_id=1				,							vocab_size=1000				,							hidden_act='''quick_gelu'''				,							projection_dim=512				,							)
        _UpperCAmelCase           =   CLIPTextModel(_UpperCamelCase					)
        _UpperCAmelCase           =   CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''					)

        _UpperCAmelCase           =   {
            '''unet''': model.eval(),
            '''vae''': vae.eval(),
            '''scheduler''': scheduler,
            '''text_encoder''': text_encoder,
            '''tokenizer''': tokenizer,
        }

        return components





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase=0					):
        if str(_UpperCamelCase					).startswith('''mps'''					):
           _UpperCAmelCase           =   torch.manual_seed(_UpperCamelCase					)
        else:
           _UpperCAmelCase           =   torch.Generator(device=_UpperCamelCase					).manual_seed(_UpperCamelCase					)
        _UpperCAmelCase           =   {
            '''prompt''': '''A painting of a squirrel eating a burger''',
            '''image''': self.dummy_image.cpu(),
            '''generator''': generator,
            '''num_inference_steps''': 2,
            '''output_type''': '''numpy''',
        }
        return inputs





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   '''cpu'''

        _UpperCAmelCase           =   self.get_dummy_components()
        _UpperCAmelCase           =   self.pipeline_class(**_UpperCamelCase					)
        pipe.to(_UpperCamelCase					)
        pipe.set_progress_bar_config(disable=_UpperCamelCase					)

        _UpperCAmelCase           =   self.get_dummy_inputs(_UpperCamelCase					)
        _UpperCAmelCase           =   pipe(**_UpperCamelCase					).images
        _UpperCAmelCase           =   image[0, -3:, -3:, -1]

        self.assertEqual(image.shape				,							(1, 256, 256, 3)					)
        _UpperCAmelCase           =   np.array(
            [0.47222412, 0.41921633, 0.44717434, 0.46874192, 0.42588258, 0.46150726, 0.4677534, 0.45583832, 0.48579055]					)
        _UpperCAmelCase           =   np.abs(image_slice.flatten() - expected_slice					).max()
        self.assertLessEqual(_UpperCamelCase				,							1e-3					)





     def 		UpperCamelCase(     self					):
        super().test_attention_slicing_forward_pass(expected_max_diff=7e-3					)





     def 		UpperCamelCase(     self					):
        super().test_cpu_offload_forward_pass(expected_max_diff=3e-3					)





     def 		UpperCamelCase(     self					):
        super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3					)





     def 		UpperCamelCase(     self					):
        super().test_inference_batch_single_identical(expected_max_diff=7e-3					)





     def 		UpperCamelCase(     self					):
        super().test_pt_np_pil_outputs_equivalent(expected_max_diff=3e-3					)





     def 		UpperCamelCase(     self					):
        super().test_save_load_local(expected_max_difference=3e-3					)





     def 		UpperCamelCase(     self					):
        super().test_save_load_optional_components(expected_max_difference=3e-3					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   [
            '''DDIMScheduler''',
            '''DDPMScheduler''',
            '''PNDMScheduler''',
            '''HeunDiscreteScheduler''',
            '''EulerAncestralDiscreteScheduler''',
            '''KDPM2DiscreteScheduler''',
            '''KDPM2AncestralDiscreteScheduler''',
            '''DPMSolverSDEScheduler''',
        ]
        _UpperCAmelCase           =   self.get_dummy_components()
        _UpperCAmelCase           =   self.pipeline_class(**_UpperCamelCase					)

        # make sure that PNDM does not need warm-up
        pipe.scheduler.register_to_config(skip_prk_steps=_UpperCamelCase					)

        pipe.to(_UpperCamelCase					)
        pipe.set_progress_bar_config(disable=_UpperCamelCase					)
        _UpperCAmelCase           =   self.get_dummy_inputs(_UpperCamelCase					)
        _UpperCAmelCase           =   2

        _UpperCAmelCase           =   []
        for scheduler_enum in KarrasDiffusionSchedulers:
           if scheduler_enum.name in skip_schedulers:
              # no sigma schedulers are not supported
              # no schedulers
              continue

           _UpperCAmelCase           =   getattr(_UpperCamelCase				,							scheduler_enum.name					)
           _UpperCAmelCase           =   scheduler_cls.from_config(pipe.scheduler.config					)
           _UpperCAmelCase           =   pipe(**_UpperCamelCase					)[0]
           outputs.append(_UpperCamelCase					)

        assert check_same_shape(_UpperCamelCase					)




@require_torch_gpu
@slow
class 			__UpperCamelCase    (							unittest.TestCase    ):





     def 		UpperCamelCase(     self					):
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   torch.manual_seed(33					)

        _UpperCAmelCase           =   StableDiffusionPipeline.from_pretrained('''CompVis/stable-diffusion-v1-4'''				,							torch_dtype=torch.floataa					)
        pipe.to('''cuda'''					)

        _UpperCAmelCase           =   StableDiffusionLatentUpscalePipeline.from_pretrained(
            '''stabilityai/sd-x2-latent-upscaler'''				,							torch_dtype=torch.floataa					)
        upscaler.to('''cuda'''					)

        _UpperCAmelCase           =   '''a photo of an astronaut high resolution, unreal engine, ultra realistic'''

        _UpperCAmelCase           =   pipe(_UpperCamelCase				,							generator=_UpperCamelCase				,							output_type='''latent'''					).images

        _UpperCAmelCase           =   upscaler(
            prompt=_UpperCamelCase				,							image=_UpperCamelCase				,							num_inference_steps=20				,							guidance_scale=0				,							generator=_UpperCamelCase				,							output_type='''np'''				,							).images[0]

        _UpperCAmelCase           =   load_numpy(
            '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/astronaut_1024.npy'''					)
        assert np.abs((expected_image - image).mean()					) < 5e-2





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   torch.manual_seed(33					)

        _UpperCAmelCase           =   StableDiffusionLatentUpscalePipeline.from_pretrained(
            '''stabilityai/sd-x2-latent-upscaler'''				,							torch_dtype=torch.floataa					)
        upscaler.to('''cuda'''					)

        _UpperCAmelCase           =   '''the temple of fire by Ross Tran and Gerardo Dottori, oil on canvas'''

        _UpperCAmelCase           =   load_image(
            '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_512.png'''					)

        _UpperCAmelCase           =   upscaler(
            prompt=_UpperCamelCase				,							image=_UpperCamelCase				,							num_inference_steps=20				,							guidance_scale=0				,							generator=_UpperCamelCase				,							output_type='''np'''				,							).images[0]

        _UpperCAmelCase           =   load_numpy(
            '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_1024.npy'''					)
        assert np.abs((expected_image - image).max()					) < 5e-2

"""simple docstring"""




import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A__ :       List[Any]				= 1_6
A__ :       Union[str, Any]				= 3_2




def 			_lowerCAmelCase		(		_UpperCamelCase			,					_UpperCamelCase = 16    ):






  """simple docstring"""



  _lowercase:      Any   			=  AutoTokenizer.from_pretrained('''bert-base-cased'''    )
  _lowercase:      List[str]   			=  load_dataset('''glue'''			,					'''mrpc'''    )

  def tokenize_function(_UpperCamelCase    ):
    # max_length=None => use the model max length (it's actually the default)
    _lowercase:      Union[str, Any]   			=  tokenizer(examples['''sentence1''']			,					examples['''sentence2''']			,					truncation=_UpperCamelCase			,					max_length=_UpperCamelCase    )
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    _lowercase:      Tuple   			=  datasets.map(
        _UpperCamelCase			,					batched=_UpperCamelCase			,					remove_columns=['''idx''', '''sentence1''', '''sentence2''']			,					)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  _lowercase:      Optional[Any]   			=  tokenized_datasets.rename_column('''label'''			,					'''labels'''    )

  def collate_fn(_UpperCamelCase    ):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    _lowercase:      Union[str, Any]   			=  128 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      _lowercase:      int   			=  16
    elif accelerator.mixed_precision != "no":
      _lowercase:      Tuple   			=  8
    else:
      _lowercase:      str   			=  None

    return tokenizer.pad(
        _UpperCamelCase			,					padding='''longest'''			,					max_length=_UpperCamelCase			,					pad_to_multiple_of=_UpperCamelCase			,					return_tensors='''pt'''			,					)

  # Instantiate dataloaders.
  _lowercase:      List[Any]   			=  DataLoader(
      tokenized_datasets['''train''']			,					shuffle=_UpperCamelCase			,					collate_fn=_UpperCamelCase			,					batch_size=_UpperCamelCase    )
  _lowercase:      int   			=  DataLoader(
      tokenized_datasets['''validation''']			,					shuffle=_UpperCamelCase			,					collate_fn=_UpperCamelCase			,					batch_size=_UpperCamelCase    )

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  A__ :       List[Any]				= mocked_dataloaders  # noqa: F811




def 			_lowerCAmelCase		(		_UpperCamelCase			,					_UpperCamelCase    ):






  """simple docstring"""



  if os.environ.get('''TESTING_MOCKED_DATALOADERS'''			,					_UpperCamelCase    ) == "1":
    _lowercase:      Tuple   			=  2
  # Initialize accelerator
  _lowercase:      Optional[Any]   			=  Accelerator(cpu=args.cpu			,					mixed_precision=args.mixed_precision    )
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  _lowercase:      Optional[Any]   			=  config['''lr''']
  _lowercase:      Tuple   			=  int(config['''num_epochs''']    )
  _lowercase:      Any   			=  int(config['''seed''']    )
  _lowercase:      List[Any]   			=  int(config['''batch_size''']    )

  _lowercase:      Optional[int]   			=  evaluate.load('''glue'''			,					'''mrpc'''    )

  # New Code #
  # We now can define an inner training loop function. It should take a batch size as the only parameter,
  # and build the dataloaders in there.
  # It also gets our decorator
  @find_executable_batch_size(starting_batch_size=_UpperCamelCase    )
  def inner_training_loop(_UpperCamelCase    ):
    # And now just move everything below under this function
    # We need to bring in the Accelerator object from earlier
    nonlocal accelerator
    # And reset all of its attributes that could hold onto any memory:
    accelerator.free_memory()

    # Then we can declare the model, optimizer, and everything else:
    set_seed(_UpperCamelCase    )

    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    _lowercase:      Optional[int]   			=  AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''			,					return_dict=_UpperCamelCase    )

    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
    _lowercase:      List[Any]   			=  model.to(accelerator.device    )

    # Instantiate optimizer
    _lowercase:      List[str]   			=  AdamW(params=model.parameters()			,					lr=_UpperCamelCase    )
    _lowercase			,					_lowercase:      Tuple   			=  get_dataloaders(_UpperCamelCase			,					_UpperCamelCase    )

    # Instantiate scheduler
    _lowercase:      str   			=  get_linear_schedule_with_warmup(
        optimizer=_UpperCamelCase			,					num_warmup_steps=100			,					num_training_steps=(len(_UpperCamelCase    ) * num_epochs)			,					)

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    _lowercase			,					_lowercase			,					_lowercase			,					_lowercase			,					_lowercase:      Union[str, Any]   			=  accelerator.prepare(
        _UpperCamelCase			,					_UpperCamelCase			,					_UpperCamelCase			,					_UpperCamelCase			,					_UpperCamelCase    )

    # Now we train the model
    for epoch in range(_UpperCamelCase    ):
      model.train()
      for step, batch in enumerate(_UpperCamelCase    ):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device    )
        _lowercase:      Tuple   			=  model(**_UpperCamelCase    )
        _lowercase:      Union[str, Any]   			=  outputs.loss
        accelerator.backward(_UpperCamelCase    )
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

      model.eval()
      for step, batch in enumerate(_UpperCamelCase    ):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device    )
        with torch.no_grad():
          _lowercase:      str   			=  model(**_UpperCamelCase    )
        _lowercase:      int   			=  outputs.logits.argmax(dim=-1    )
        _lowercase			,					_lowercase:      List[str]   			=  accelerator.gather_for_metrics((predictions, batch['''labels'''])    )
        metric.add_batch(
            predictions=_UpperCamelCase			,					references=_UpperCamelCase			,					)

      _lowercase:      int   			=  metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(f'''epoch {epoch}:'''			,					_UpperCamelCase    )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
  inner_training_loop()




def 			_lowerCAmelCase		(		):






  """simple docstring"""



  _lowercase:      Dict   			=  argparse.ArgumentParser(description='''Simple example of training script.'''    )
  parser.add_argument(
      '''--mixed_precision'''			,					type=_UpperCamelCase			,					default=_UpperCamelCase			,					choices=['''no''', '''fp16''', '''bf16''', '''fp8''']			,					help='''Whether to use mixed precision. Choose'''
      '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
      '''and an Nvidia Ampere GPU.'''			,					)
  parser.add_argument('''--cpu'''			,					action='''store_true'''			,					help='''If passed, will train on the CPU.'''    )
  _lowercase:      Tuple   			=  parser.parse_args()
  _lowercase:      str   			=  {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
  training_function(_UpperCamelCase			,					_UpperCamelCase    )


if __name__ == "__main__":
  main()
