






import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


UpperCAmelCase__      			=						16
UpperCAmelCase__      			=						32







def     a_   (__A      )				->						Union[str, Any]:



					"""simple docstring"""
					return int(x / 2**20      )




class 							snake_case_       :

						"""simple docstring"""



						def __enter__(self:					Optional[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											gc.collect()
											torch.cuda.empty_cache()
											torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
											__a					:						List[Any]      =					torch.cuda.memory_allocated()
											return self



						def __exit__(self:					List[str] ,					*__UpperCAmelCase:					str   )    ->     int:




											'''simple docstring'''



											gc.collect()
											torch.cuda.empty_cache()
											__a					:						Any      =					torch.cuda.memory_allocated()
											__a					:						Optional[Any]      =					torch.cuda.max_memory_allocated()
											__a					:						Optional[Any]      =					bamb(self.end - self.begin   )
											__a					:						Tuple      =					bamb(self.peak - self.begin   )
											# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")







def     a_   (__A				,       __A = 16				,       __A = "bert-base-cased"				,       __A = 320				,       __A = 160				,       )				->						Tuple:



					"""simple docstring"""
					__a					:						Any      =					AutoTokenizer.from_pretrained(__A      )
					__a					:						Optional[int]      =					load_dataset(
					    "glue"				,       "mrpc"				,       split={"train": f'train[:{n_train}]', "validation": f'validation[:{n_val}]'}      )

					def tokenize_function(__A      ):
										# max_length=None => use the model max length (it's actually the default)
										__a					:						List[Any]      =					tokenizer(examples["sentence1"]				,       examples["sentence2"]				,       truncation=__A				,       max_length=__A      )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					__a					:						List[Any]      =					datasets.map(
					    __A				,       batched=__A				,       remove_columns=["idx", "sentence1", "sentence2"]				,       load_from_cache_file=__A      )

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					__a					:						Tuple      =					tokenized_datasets.rename_column("label"				,       "labels"      )

					def collate_fn(__A      ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										if accelerator.distributed_type == DistributedType.TPU:
															return tokenizer.pad(__A				,       padding="max_length"				,       max_length=128				,       return_tensors="pt"      )
										return tokenizer.pad(__A				,       padding="longest"				,       return_tensors="pt"      )

					# Instantiate dataloaders.
					__a					:						Union[str, Any]      =					DataLoader(
					    tokenized_datasets["train"]				,       shuffle=__A				,       collate_fn=__A				,       batch_size=__A      )
					__a					:						List[Any]      =					DataLoader(
					    tokenized_datasets["validation"]				,       shuffle=__A				,       collate_fn=__A				,       batch_size=__A      )

					return train_dataloader, eval_dataloader







def     a_   (__A				,       __A      )				->						List[str]:



					"""simple docstring"""
					# Initialize accelerator
					__a					:						Dict      =					Accelerator()

					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					__a					:						Tuple      =					config["lr"]
					__a					:						int      =					int(config["num_epochs"]      )
					__a					:						List[Any]      =					int(config["seed"]      )
					__a					:						Union[str, Any]      =					int(config["batch_size"]      )
					__a					:						str      =					args.model_name_or_path

					set_seed(__A      )
					__a      ,			__a					:						int      =					get_dataloaders(__A				,       __A				,       __A				,       args.n_train				,       args.n_val      )

					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					__a					:						Tuple      =					AutoModelForSequenceClassification.from_pretrained(__A				,       return_dict=__A      )

					# Instantiate optimizer
					__a					:						Any      =					(
					    AdamW
					    if accelerator.state.deepspeed_plugin is None
					    or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config
					    else DummyOptim
					)
					__a					:						Dict      =					optimizer_cls(params=model.parameters()				,       lr=__A      )

					if accelerator.state.deepspeed_plugin is not None:
										__a					:						str      =					accelerator.state.deepspeed_plugin.deepspeed_config[
										    "gradient_accumulation_steps"
										]
					else:
										__a					:						List[str]      =					1
					__a					:						Any      =					(len(__A      ) * num_epochs) // gradient_accumulation_steps

					# Instantiate scheduler
					if (
					    accelerator.state.deepspeed_plugin is None
					    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
					):
										__a					:						Any      =					get_linear_schedule_with_warmup(
										    optimizer=__A				,       num_warmup_steps=0				,       num_training_steps=__A				,       )
					else:
										__a					:						Any      =					DummyScheduler(__A				,       total_num_steps=__A				,       warmup_num_steps=0      )

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					__a      ,			__a      ,			__a      ,			__a      ,			__a					:						Optional[Any]      =					accelerator.prepare(
					    __A				,       __A				,       __A				,       __A				,       __A      )

					# We need to keep track of how many total steps we have iterated over
					__a					:						Optional[Any]      =					0
					# We also need to keep track of the stating epoch so files are named properly
					__a					:						str      =					0

					# Now we train the model
					__a					:						Optional[Any]      =					{}
					for epoch in range(__A				,       __A      ):
										with TorchTracemalloc() as tracemalloc:
															model.train()
															for step, batch in enumerate(__A      ):
																				__a					:						Dict      =					model(**__A      )
																				__a					:						Union[str, Any]      =					outputs.loss
																				__a					:						Optional[Any]      =					loss / gradient_accumulation_steps
																				accelerator.backward(__A      )
																				if step % gradient_accumulation_steps == 0:
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()

																				overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin      )      )      )
										accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used      )      )
										accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked      )      )
										accelerator.print(
										    "Total Peak Memory consumed during the train (max): {}".format(
										        tracemalloc.peaked + bamb(tracemalloc.begin      )      )      )
										__a					:						int      =					tracemalloc.peaked + bamb(tracemalloc.begin      )
										if args.peak_memory_upper_bound is not None:
															assert (
															    train_total_peak_memory[f'epoch-{epoch}'] <= args.peak_memory_upper_bound
															), "Peak memory usage exceeded the upper bound"

					accelerator.wait_for_everyone()
					if accelerator.is_main_process:
										with open(os.path.join(args.output_dir				,       "peak_memory_utilization.json"      )				,       "w"      ) as f:
															json.dump(__A				,       __A      )







def     a_   ()				->						str:



					"""simple docstring"""
					__a					:						Union[str, Any]      =					argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage."      )
					parser.add_argument(
					    "--model_name_or_path"				,       type=__A				,       default="bert-base-cased"				,       help="Path to pretrained model or model identifier from huggingface.co/models."				,       required=__A				,       )
					parser.add_argument(
					    "--output_dir"				,       type=__A				,       default="."				,       help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."				,       )
					parser.add_argument(
					    "--peak_memory_upper_bound"				,       type=__A				,       default=__A				,       help="The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value."				,       )
					parser.add_argument(
					    "--n_train"				,       type=__A				,       default=320				,       help="Number of training examples to use."				,       )
					parser.add_argument(
					    "--n_val"				,       type=__A				,       default=160				,       help="Number of validation examples to use."				,       )
					parser.add_argument(
					    "--num_epochs"				,       type=__A				,       default=1				,       help="Number of train epochs."				,       )
					__a					:						str      =					parser.parse_args()
					__a					:						Any      =					{"lr": 2e-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16}
					training_function(__A				,       __A      )


if __name__ == "__main__":
			main()













import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					str ,					*__UpperCAmelCase:					Optional[int] ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Dict=None ,					**__UpperCAmelCase:					Tuple   )    ->     Dict:




											'''simple docstring'''



											super().__init__(*__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						str      =					eval_examples
											__a					:						List[Any]      =					post_process_function



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Optional[Dataset] = None ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Optional[List[str]] = None ,					__UpperCAmelCase:					str = "eval" ,					**__UpperCAmelCase:					Any ,					)    ->     Dict[str, float]:




											'''simple docstring'''



											__a					:						Optional[int]      =					gen_kwargs.copy()
											__a					:						Union[str, Any]      =					(
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"   ) is not None else self.args.generation_max_length
											)
											__a					:						Dict      =					(
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"   ) is not None else self.args.generation_num_beams
											)
											__a					:						Optional[Any]      =					gen_kwargs

											__a					:						Union[str, Any]      =					self.eval_dataset if eval_dataset is None else eval_dataset
											__a					:						Optional[int]      =					self.get_eval_dataloader(__UpperCAmelCase   )
											__a					:						Tuple      =					self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						int      =					self.compute_metrics
											__a					:						Dict      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						str      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						str      =					eval_loop(
																    __UpperCAmelCase ,					description="Evaluation" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						Union[str, Any]      =					compute_metrics
											__a					:						str      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__a					:						Union[str, Any]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )
																__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()   ):
																					if not key.startswith(f'{metric_key_prefix}_'   ):
																										__a					:						int      =					metrics.pop(__UpperCAmelCase   )

																metrics.update(output.metrics   )
											else:
																__a					:						Optional[int]      =					output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(__UpperCAmelCase   )

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()   )

											__a					:						Tuple      =					self.callback_handler.on_evaluate(self.args ,					self.state ,					self.control ,					__UpperCAmelCase   )
											return metrics



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					Union[str, Any]=None ,					__UpperCAmelCase:					str = "test" ,					**__UpperCAmelCase:					List[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Any      =					gen_kwargs.copy()

											__a					:						Tuple      =					self.get_test_dataloader(__UpperCAmelCase   )

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						Optional[Any]      =					self.compute_metrics
											__a					:						int      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						Dict      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						Optional[Any]      =					eval_loop(
																    __UpperCAmelCase ,					description="Prediction" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						List[Any]      =					compute_metrics

											__a					:						Optional[Any]      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__a					:						Optional[int]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					"predict"   )
											__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()   ):
																if not key.startswith(f'{metric_key_prefix}_'   ):
																					__a					:						List[str]      =					metrics.pop(__UpperCAmelCase   )
											metrics.update(output.metrics   )
											return PredictionOutput(predictions=predictions.predictions ,					label_ids=predictions.label_ids ,					metrics=__UpperCAmelCase   )





