






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


SCREAMING_SNAKE_CASE__				     =			16
SCREAMING_SNAKE_CASE__				     =			32

def        UpperCAmelCase__		(						lowerCamelCase_       :       Accelerator   ,  lowerCamelCase_       :       int = 1_6				):
			__a						:  List[Any]									=     AutoTokenizer.from_pretrained('bert-base-cased'				)
			__a						:  int									=     load_dataset('glue'   ,  'mrpc'				)

			def tokenize_function(lowerCamelCase_       :       int				):
						# max_length=None => use the model max length (it's actually the default)
						__a						:  Dict									=     tokenizer(examples['sentence1']   ,  examples['sentence2']   ,  truncation=lowerCamelCase_   ,  max_length=lowerCamelCase_				)
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						__a						:  str									=     datasets.map(
						    lowerCamelCase_   ,  batched=lowerCamelCase_   ,  remove_columns=['idx', 'sentence1', 'sentence2']   ,  )

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			__a						:  Union[str, Any]									=     tokenized_datasets.rename_column('label'   ,  'labels'				)

			def collate_fn(lowerCamelCase_       :       Union[str, Any]				):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						__a						:  Tuple									=     1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									__a						:  Any									=     1_6
						elif accelerator.mixed_precision != "no":
									__a						:  Optional[Any]									=     8
						else:
									__a						:  str									=     None

						return tokenizer.pad(
						    lowerCamelCase_   ,  padding='longest'   ,  max_length=lowerCamelCase_   ,  pad_to_multiple_of=lowerCamelCase_   ,  return_tensors='pt'   ,  )

			# Instantiate dataloaders.
			__a						:  Union[str, Any]									=     DataLoader(
			    tokenized_datasets['train']   ,  shuffle=lowerCamelCase_   ,  collate_fn=lowerCamelCase_   ,  batch_size=lowerCamelCase_				)
			__a						:  Dict									=     DataLoader(
			    tokenized_datasets['validation']   ,  shuffle=lowerCamelCase_   ,  collate_fn=lowerCamelCase_   ,  batch_size=lowerCamelCase_				)

			return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							SCREAMING_SNAKE_CASE__				     =			mocked_dataloaders  # noqa: F811

def        UpperCAmelCase__		(						lowerCamelCase_       :       Tuple   ,  lowerCamelCase_       :       Dict				):
			# For testing only
			if os.environ.get('TESTING_MOCKED_DATALOADERS'   ,  lowerCamelCase_				) == "1":
						__a						:  List[str]									=     2
			# New Code #
			__a						:  Dict									=     int(args.gradient_accumulation_steps				)
			# Initialize accelerator
			__a						:  Optional[Any]									=     Accelerator(
			    cpu=args.cpu   ,  mixed_precision=args.mixed_precision   ,  gradient_accumulation_steps=lowerCamelCase_				)
			if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
						raise NotImplementedError(
						    'Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`'				)
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			__a						:  Union[str, Any]									=     config['lr']
			__a						:  List[Any]									=     int(config['num_epochs']				)
			__a						:  Optional[int]									=     int(config['seed']				)
			__a						:  Dict									=     int(config['batch_size']				)

			__a						:  Optional[int]									=     evaluate.load('glue'   ,  'mrpc'				)

			set_seed(lowerCamelCase_				)
			__a						,					__a						:  List[Any]									=     get_dataloaders(lowerCamelCase_   ,  lowerCamelCase_				)
			# Instantiate the model (we build the model here so that the seed also control new weights initialization)
			__a						:  int									=     AutoModelForSequenceClassification.from_pretrained('bert-base-cased'   ,  return_dict=lowerCamelCase_				)

			# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
			# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
			# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
			__a						:  Any									=     model.to(accelerator.device				)

			# Instantiate optimizer
			__a						:  Dict									=     AdamW(params=model.parameters()   ,  lr=lowerCamelCase_				)

			# Instantiate scheduler
			__a						:  List[str]									=     get_linear_schedule_with_warmup(
			    optimizer=lowerCamelCase_   ,  num_warmup_steps=1_0_0   ,  num_training_steps=(len(lowerCamelCase_				) * num_epochs)   ,  )

			# Prepare everything
			# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
			# prepare method.
			__a						,					__a						,					__a						,					__a						,					__a						:  Dict									=     accelerator.prepare(
			    lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)

			# Now we train the model
			for epoch in range(lowerCamelCase_				):
						model.train()
						for step, batch in enumerate(lowerCamelCase_				):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device				)
									# New code #
									# We use the new `accumulate` context manager to perform gradient accumulation
									# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
									with accelerator.accumulate(lowerCamelCase_				):
												__a						:  Any									=     model(**lowerCamelCase_				)
												__a						:  Dict									=     output.loss
												accelerator.backward(lowerCamelCase_				)
												optimizer.step()
												lr_scheduler.step()
												optimizer.zero_grad()

						model.eval()
						for step, batch in enumerate(lowerCamelCase_				):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device				)
									with torch.no_grad():
												__a						:  Any									=     model(**lowerCamelCase_				)
									__a						:  Optional[Any]									=     outputs.logits.argmax(dim=-1				)
									__a						,					__a						:  Dict									=     accelerator.gather_for_metrics((predictions, batch['labels'])				)
									metric.add_batch(
									    predictions=lowerCamelCase_   ,  references=lowerCamelCase_   ,  )

						__a						:  Optional[int]									=     metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(f'''epoch {epoch}:'''   ,  lowerCamelCase_				)

def        UpperCAmelCase__		(						):
			__a						:  Union[str, Any]									=     argparse.ArgumentParser(description='Simple example of training script.'				)
			parser.add_argument(
			    '--mixed_precision'   ,  type=lowerCamelCase_   ,  default=lowerCamelCase_   ,  choices=['no', 'fp16', 'bf16', 'fp8']   ,  help='Whether to use mixed precision. Choose'
			    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
			    'and an Nvidia Ampere GPU.'   ,  )
			# New Code #
			parser.add_argument(
			    '--gradient_accumulation_steps'   ,  type=lowerCamelCase_   ,  default=1   ,  help='The number of minibatches to be ran before gradients are accumulated.'   ,  )
			parser.add_argument('--cpu'   ,  action='store_true'   ,  help='If passed, will train on the CPU.'				)
			__a						:  Any									=     parser.parse_args()
			__a						:  Optional[Any]									=     {'lr': 2e-5, 'num_epochs': 3, 'seed': 4_2, 'batch_size': 1_6}
			training_function(lowerCamelCase_   ,  lowerCamelCase_				)


if __name__ == "__main__":
							main()













import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


class     _a							(	UpperCAmelCase__    ,							UpperCAmelCase__    ,							unittest.TestCase     ):






  """simple docstring"""





  A_						       = StableDiffusionSAGPipeline
  A_						       = TEXT_TO_IMAGE_PARAMS
  A_						       = TEXT_TO_IMAGE_BATCH_PARAMS
  A_						       = TEXT_TO_IMAGE_IMAGE_PARAMS
  A_						       = TEXT_TO_IMAGE_IMAGE_PARAMS
  A_						       = False




  def  _UpperCAmelCase  (  self			)      ->			Optional[Any]:
     torch.manual_seed(0			)
     UpperCamelCase_						=  UNetaDConditionModel(
         block_out_channels=(32, 64)		,		layers_per_block=2		,		sample_size=32		,		in_channels=4		,		out_channels=4		,		down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')		,		up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')		,		cross_attention_dim=32		,		)
     UpperCamelCase_						=  DDIMScheduler(
         beta_start=0.0_0_0_8_5		,		beta_end=0.0_1_2		,		beta_schedule='scaled_linear'		,		clip_sample=_UpperCAmelCase		,		set_alpha_to_one=_UpperCAmelCase		,		)
     torch.manual_seed(0			)
     UpperCamelCase_						=  AutoencoderKL(
         block_out_channels=[32, 64]		,		in_channels=3		,		out_channels=3		,		down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']		,		up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']		,		latent_channels=4		,		)
     torch.manual_seed(0			)
     UpperCamelCase_						=  CLIPTextConfig(
         bos_token_id=0		,		eos_token_id=2		,		hidden_size=32		,		intermediate_size=37		,		layer_norm_eps=1e-05		,		num_attention_heads=4		,		num_hidden_layers=5		,		pad_token_id=1		,		vocab_size=1000		,		)
     UpperCamelCase_						=  CLIPTextModel(_UpperCAmelCase			)
     UpperCamelCase_						=  CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'			)

     UpperCamelCase_						=  {
         'unet': unet,
         'scheduler': scheduler,
         'vae': vae,
         'text_encoder': text_encoder,
         'tokenizer': tokenizer,
         'safety_checker': None,
         'feature_extractor': None,
     }
     return components




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase=0			)      ->			List[Any]:
     if str(_UpperCAmelCase			).startswith('mps'			):
        UpperCamelCase_						=  torch.manual_seed(_UpperCAmelCase			)
     else:
        UpperCamelCase_						=  torch.Generator(device=_UpperCAmelCase			).manual_seed(_UpperCAmelCase			)
     UpperCamelCase_						=  {
         'prompt': '.',
         'generator': generator,
         'num_inference_steps': 2,
         'guidance_scale': 1.0,
         'sag_scale': 1.0,
         'output_type': 'numpy',
     }
     return inputs




  def  _UpperCAmelCase  (  self			)      ->			Tuple:
     super().test_inference_batch_single_identical(expected_max_diff=3e-3			)




@slow
@require_torch_gpu
class     _a							(	unittest.TestCase     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Tuple:
     # clean up the VRAM after each test
     super().tearDown()
     gc.collect()
     torch.cuda.empty_cache()




  def  _UpperCAmelCase  (  self			)      ->			str:
     UpperCamelCase_						=  StableDiffusionSAGPipeline.from_pretrained('CompVis/stable-diffusion-v1-4'			)
     UpperCamelCase_						=  sag_pipe.to(_UpperCAmelCase			)
     sag_pipe.set_progress_bar_config(disable=_UpperCAmelCase			)

     UpperCamelCase_						=  '.'
     UpperCamelCase_						=  torch.manual_seed(0			)
     UpperCamelCase_						=  sag_pipe(
         [prompt]		,		generator=_UpperCAmelCase		,		guidance_scale=7.5		,		sag_scale=1.0		,		num_inference_steps=20		,		output_type='np'			)

     UpperCamelCase_						=  output.images

     UpperCamelCase_						=  image[0, -3:, -3:, -1]

     assert image.shape == (1, 512, 512, 3)
     UpperCamelCase_						=  np.array([0.1_5_6_8, 0.1_7_3_8, 0.1_6_9_5, 0.1_6_9_3, 0.1_5_0_7, 0.1_7_0_5, 0.1_5_4_7, 0.1_7_5_1, 0.1_9_4_9]			)

     assert np.abs(image_slice.flatten() - expected_slice			).max() < 5e-2




  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  StableDiffusionSAGPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'			)
     UpperCamelCase_						=  sag_pipe.to(_UpperCAmelCase			)
     sag_pipe.set_progress_bar_config(disable=_UpperCAmelCase			)

     UpperCamelCase_						=  '.'
     UpperCamelCase_						=  torch.manual_seed(0			)
     UpperCamelCase_						=  sag_pipe(
         [prompt]		,		generator=_UpperCAmelCase		,		guidance_scale=7.5		,		sag_scale=1.0		,		num_inference_steps=20		,		output_type='np'			)

     UpperCamelCase_						=  output.images

     UpperCamelCase_						=  image[0, -3:, -3:, -1]

     assert image.shape == (1, 512, 512, 3)
     UpperCamelCase_						=  np.array([0.3_4_5_9, 0.2_8_7_6, 0.2_5_3_7, 0.3_0_0_2, 0.2_6_7_1, 0.2_1_6_0, 0.3_0_2_6, 0.2_2_6_2, 0.2_3_7_1]			)

     assert np.abs(image_slice.flatten() - expected_slice			).max() < 5e-2




  def  _UpperCAmelCase  (  self			)      ->			Dict:
     UpperCamelCase_						=  StableDiffusionSAGPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base'			)
     UpperCamelCase_						=  sag_pipe.to(_UpperCAmelCase			)
     sag_pipe.set_progress_bar_config(disable=_UpperCAmelCase			)

     UpperCamelCase_						=  '.'
     UpperCamelCase_						=  torch.manual_seed(0			)
     UpperCamelCase_						=  sag_pipe(
         [prompt]		,		width=768		,		height=512		,		generator=_UpperCAmelCase		,		guidance_scale=7.5		,		sag_scale=1.0		,		num_inference_steps=20		,		output_type='np'		,		)

     UpperCamelCase_						=  output.images

     assert image.shape == (1, 512, 768, 3)





