






import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__A          = 16
__A          = 32




def    __A    (   _lowercase       ):

					'''simple docstring'''







					return int(x / 2**20       )





class   SCREAMING_SNAKE_CASE :





							"""simple docstring"""

							def __enter__(     self:      int      )			-> Optional[Any]:
												gc.collect()
												torch.cuda.empty_cache()
												torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
												_A									=       torch.cuda.memory_allocated()
												_A									=       psutil.Process()

												_A									=       self.cpu_mem_used()
												_A									=       True
												_A									=       threading.Thread(target=self.peak_monitor_func      )
												_A									=       True
												peak_monitor_thread.start()
												return self

							def 					__A       (     self:      Optional[int]      )			-> Any:
												return self.process.memory_info().rss

							def 					__A       (     self:      str      )			-> Optional[int]:
												_A									=       -1

												while True:
																	_A									=       max(self.cpu_mem_used()						,   self.cpu_peak      )

																	# can't sleep or will not catch the peak right (this comment is here on purpose)
																	# time.sleep(0.001) # 1msec

																	if not self.peak_monitoring:
																						break



							def __exit__(     self:      Optional[int]						,   *__A:      Union[str, Any]      )			-> Union[str, Any]:
												_A									=       False

												gc.collect()
												torch.cuda.empty_cache()
												_A									=       torch.cuda.memory_allocated()
												_A									=       torch.cuda.max_memory_allocated()
												_A									=       bamb(self.end - self.begin      )
												_A									=       bamb(self.peak - self.begin      )

												_A									=       self.cpu_mem_used()
												_A									=       bamb(self.cpu_end - self.cpu_begin      )
												_A									=       bamb(self.cpu_peak - self.cpu_begin      )
												# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
				from accelerate.test_utils.training import mocked_dataloaders

				__A          = mocked_dataloaders  # noqa: F811




def    __A    (   _lowercase    ,	_lowercase       ):

					'''simple docstring'''







					if os.environ.get('''TESTING_MOCKED_DATALOADERS'''    ,	_lowercase       ) == "1":
										_A									=       2

					# New Code #
					# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
					_A									=       FullyShardedDataParallelPlugin(
					    state_dict_config=FullStateDictConfig(offload_to_cpu=_lowercase    ,	ranka_only=_lowercase       )    ,	optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_lowercase    ,	ranka_only=_lowercase       )    ,	)

					# Initialize accelerator
					if args.with_tracking:
										_A									=       Accelerator(
										    cpu=args.cpu    ,	mixed_precision=args.mixed_precision    ,	log_with='''wandb'''    ,	project_dir=args.logging_dir    ,	fsdp_plugin=_lowercase    ,	)
					else:
										_A									=       Accelerator(fsdp_plugin=_lowercase       )
					accelerator.print(accelerator.distributed_type       )

					if hasattr(args.checkpointing_steps    ,	'''isdigit'''       ):
										if args.checkpointing_steps == "epoch":
															_A									=       args.checkpointing_steps
										elif args.checkpointing_steps.isdigit():
															_A									=       int(args.checkpointing_steps       )
										else:
															raise ValueError(
															    f"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""       )
					else:
										_A									=       None
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					_A									=       config['''lr''']
					_A									=       int(config['''num_epochs''']       )
					_A									=       int(config['''seed''']       )
					_A									=       int(config['''batch_size''']       )

					# We need to initialize the trackers we use, and also store our configuration
					if args.with_tracking:
										_A									=       vars(_lowercase       )
										accelerator.init_trackers('''fsdp_glue_no_trainer'''    ,	_lowercase       )

					_A									=       AutoTokenizer.from_pretrained(args.model_name_or_path       )
					_A									=       load_dataset('''glue'''    ,	'''mrpc'''       )
					_A									=       evaluate.load('''glue'''    ,	'''mrpc'''       )

					def tokenize_function(_lowercase       ):
										# max_length=None => use the model max length (it's actually the default)
										_A									=       tokenizer(examples['''sentence1''']    ,	examples['''sentence2''']    ,	truncation=_lowercase    ,	max_length=_lowercase       )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										_A									=       datasets.map(
										    _lowercase    ,	batched=_lowercase    ,	remove_columns=['''idx''', '''sentence1''', '''sentence2''']    ,	)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					_A									=       tokenized_datasets.rename_column('''label'''    ,	'''labels'''       )

					# If the batch size is too big we use gradient accumulation
					_A									=       1
					if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
										_A									=       batch_size // MAX_GPU_BATCH_SIZE
										_A									=       MAX_GPU_BATCH_SIZE

					def collate_fn(_lowercase       ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										_A									=       1_28 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															_A									=       16
										elif accelerator.mixed_precision != "no":
															_A									=       8
										else:
															_A									=       None

										return tokenizer.pad(
										    _lowercase    ,	padding='''longest'''    ,	max_length=_lowercase    ,	pad_to_multiple_of=_lowercase    ,	return_tensors='''pt'''    ,	)

					# Instantiate dataloaders.
					_A									=       DataLoader(
					    tokenized_datasets['''train''']    ,	shuffle=_lowercase    ,	collate_fn=_lowercase    ,	batch_size=_lowercase       )
					_A									=       DataLoader(
					    tokenized_datasets['''validation''']    ,	shuffle=_lowercase    ,	collate_fn=_lowercase    ,	batch_size=_lowercase       )

					set_seed(_lowercase       )

					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					_A									=       AutoModelForSequenceClassification.from_pretrained(
					    args.model_name_or_path    ,	return_dict=_lowercase    ,	low_cpu_mem_usage=_lowercase       )

					# New Code #
					# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
					_A									=       accelerator.prepare(_lowercase       )
					accelerator.print(_lowercase       )

					# Instantiate optimizer
					# New Code #
					# For FSDP feature, at present it doesn't support multiple parameter groups,
					# so we need to create a single parameter group for the whole model
					_A									=       torch.optim.AdamW(params=model.parameters()    ,	lr=_lowercase    ,	weight_decay=2e-4       )

					# Instantiate scheduler
					_A									=       get_linear_schedule_with_warmup(
					    optimizer=_lowercase    ,	num_warmup_steps=10    ,	num_training_steps=(len(_lowercase       ) * num_epochs) // gradient_accumulation_steps    ,	)

					# New Code #
					# For FSDP feature, prepare everything except the model as we have already prepared the model
					# before creating the optimizer
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					_A							,_A							,_A							,_A									=       accelerator.prepare(
					    _lowercase    ,	_lowercase    ,	_lowercase    ,	_lowercase       )

					_A									=       0

					# Potentially load in the weights and states from a previous save
					if args.resume_from_checkpoint:
										if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
															accelerator.print(f"""Resumed from checkpoint: {args.resume_from_checkpoint}"""       )
															accelerator.load_state(args.resume_from_checkpoint       )
															_A									=       os.path.basename(args.resume_from_checkpoint       )
										else:
															# Get the most recent checkpoint
															_A									=       [f.name for f in os.scandir(os.getcwd()       ) if f.is_dir()]
															dirs.sort(key=os.path.getctime       )
															_A									=       dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
										# Extract `epoch_{i}` or `step_{i}`
										_A									=       os.path.splitext(_lowercase       )[0]

										if "epoch" in training_difference:
															num_epochs -= int(training_difference.replace('''epoch_'''    ,	''''''       )       )
															_A									=       None
										else:
															_A									=       int(training_difference.replace('''step_'''    ,	''''''       )       )
															num_epochs -= resume_step // len(_lowercase       )
															# If resuming by step, we also need to know exactly how far into the DataLoader we went
															_A									=       (num_epochs * len(_lowercase       )) - resume_step

    # Now we train the model
					for epoch in range(_lowercase       ):
										# New Code #
										# context manager to track the peak memory usage during the training epoch
										with TorchTracemalloc() as tracemalloc:
															model.train()
															if args.with_tracking:
																				_A									=       0
															for step, batch in enumerate(_lowercase       ):
																				# We need to skip steps until we reach the resumed step
																				if args.resume_from_checkpoint and epoch == 0:
																									if resume_step is not None and step < resume_step:
																														pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device       )
																				_A									=       model(**_lowercase       )
																				_A									=       outputs.loss
																				_A									=       loss / gradient_accumulation_steps
																				# We keep track of the loss at each epoch
																				if args.with_tracking:
																									total_loss += loss.detach().float()
																				accelerator.backward(_lowercase       )
																				if step % gradient_accumulation_steps == 0:
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# accelerator.print(lr_scheduler.get_lr())

																				overall_step += 1

																				if isinstance(_lowercase    ,	_lowercase       ):
																									_A									=       f"""step_{overall_step}"""
																									if overall_step % checkpointing_steps == 0:
																														if args.output_dir is not None:
																																			_A									=       os.path.join(args.output_dir    ,	_lowercase       )
																														accelerator.save_state(_lowercase       )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin       )       )       )
										accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used       )       )
										accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked       )       )
										accelerator.print(
										    '''Total Peak Memory consumed during the train (max): {}'''.format(
										        tracemalloc.peaked + bamb(tracemalloc.begin       )       )       )
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin       ),
															    }    ,	step=_lowercase    ,	)

										# New Code #
										# context manager to track the peak memory usage during the evaluation
										with TorchTracemalloc() as tracemalloc:
															model.eval()
															for step, batch in enumerate(_lowercase       ):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device       )
																				with torch.no_grad():
																									_A									=       model(**_lowercase       )
																				_A									=       outputs.logits.argmax(dim=-1       )
																				_A							,_A									=       accelerator.gather_for_metrics((predictions, batch['''labels'''])       )
																				metric.add_batch(
																				    predictions=_lowercase    ,	references=_lowercase    ,	)

															_A									=       metric.compute()
															# Use accelerator.print to print only on the main process.
															accelerator.print(f"""epoch {epoch}:"""    ,	_lowercase       )
															if args.with_tracking:
																				accelerator.log(
																				    {
																				        '''accuracy''': eval_metric['''accuracy'''],
																				        '''f1''': eval_metric['''f1'''],
																				        '''train_loss''': total_loss.item() / len(_lowercase       ),
																				    }    ,	step=_lowercase    ,	)

															if checkpointing_steps == "epoch":
																				_A									=       f"""epoch_{epoch}"""
																				if args.output_dir is not None:
																									_A									=       os.path.join(args.output_dir    ,	_lowercase       )
																				accelerator.save_state(_lowercase       )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin       )       )       )
										accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used       )       )
										accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked       )       )
										accelerator.print(
										    '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin       )       )       )
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin       ),
															    }    ,	step=_lowercase    ,	)

					if args.with_tracking:
										accelerator.end_training()




def    __A    (   ):

					'''simple docstring'''







					_A									=       argparse.ArgumentParser(description='''Simple example of training script.'''       )
					parser.add_argument(
					    '''--mixed_precision'''    ,	type=_lowercase    ,	default=_lowercase    ,	choices=['''no''', '''fp16''', '''bf16''', '''fp8''']    ,	help='''Whether to use mixed precision. Choose'''
					    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
					    '''and an Nvidia Ampere GPU.'''    ,	)
					parser.add_argument('''--cpu'''    ,	action='''store_true'''    ,	help='''If passed, will train on the CPU.'''       )
					parser.add_argument(
					    '''--checkpointing_steps'''    ,	type=_lowercase    ,	default=_lowercase    ,	help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''    ,	)
					parser.add_argument(
					    '''--resume_from_checkpoint'''    ,	type=_lowercase    ,	default=_lowercase    ,	help='''If the training should continue from a checkpoint folder.'''    ,	)
					parser.add_argument(
					    '''--with_tracking'''    ,	action='''store_true'''    ,	help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''    ,	)
					parser.add_argument(
					    '''--output_dir'''    ,	type=_lowercase    ,	default='''.'''    ,	help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''    ,	)
					parser.add_argument(
					    '''--logging_dir'''    ,	type=_lowercase    ,	default='''logs'''    ,	help='''Location on where to store experiment tracking logs`'''    ,	)
					parser.add_argument(
					    '''--model_name_or_path'''    ,	type=_lowercase    ,	help='''Path to pretrained model or model identifier from huggingface.co/models.'''    ,	required=_lowercase    ,	)
					_A									=       parser.parse_args()
					_A									=       {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
					training_function(_lowercase    ,	_lowercase       )


if __name__ == "__main__":
				main()











import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A          = logging.get_logger(__name__)

__A          = {}





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    "fsmt_decoder"

							def __init__(     self:      List[Any]						,   __A:      Dict=0						,   __A:      str=0      )			-> Optional[int]:
												super().__init__()
												_A									=       vocab_size
												_A									=       bos_token_id





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    "fsmt"
							A_       =    {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

							def __init__(     self:      Optional[Any]						,   __A:      Optional[Any]=["en", "de"]						,   __A:      List[str]=4_20_24						,   __A:      Any=4_20_24						,   __A:      Optional[int]="relu"						,   __A:      List[str]=10_24						,   __A:      str=2_00						,   __A:      Tuple=10_24						,   __A:      List[str]=40_96						,   __A:      List[Any]=12						,   __A:      Union[str, Any]=16						,   __A:      List[str]=0.0						,   __A:      int=40_96						,   __A:      Union[str, Any]=12						,   __A:      List[str]=16						,   __A:      Union[str, Any]=0.0						,   __A:      int=0.0						,   __A:      str=0.1						,   __A:      int=0.0						,   __A:      Union[str, Any]=0.02						,   __A:      Tuple=2						,   __A:      int=True						,   __A:      Dict=True						,   __A:      int=False						,   __A:      Any=5						,   __A:      Tuple=1.0						,   __A:      Optional[int]=False						,   __A:      List[Any]=True						,   __A:      Tuple=1						,   __A:      List[Any]=0						,   __A:      Any=2						,   __A:      Optional[Any]=2						,   **__A:      Optional[Any]						,   )			-> Tuple:
												_A									=       langs
												_A									=       src_vocab_size
												_A									=       tgt_vocab_size
												_A									=       d_model  # encoder_embed_dim and decoder_embed_dim

												_A									=       encoder_ffn_dim
												_A									=       _A									=       encoder_layers
												_A									=       encoder_attention_heads
												_A									=       encoder_layerdrop
												_A									=       decoder_layerdrop
												_A									=       decoder_ffn_dim
												_A									=       decoder_layers
												_A									=       decoder_attention_heads
												_A									=       max_position_embeddings
												_A									=       init_std  # Normal(0, this parameter)
												_A									=       activation_function

												_A									=       DecoderConfig(vocab_size=__A						,   bos_token_id=__A      )
												if "decoder" in common_kwargs:
																	del common_kwargs["decoder"]

												_A									=       scale_embedding  # scale factor will be sqrt(d_model) if True

												# 3 Types of Dropout
												_A									=       attention_dropout
												_A									=       activation_dropout
												_A									=       dropout

												_A									=       use_cache
												super().__init__(
												    pad_token_id=__A						,   bos_token_id=__A						,   eos_token_id=__A						,   decoder_start_token_id=__A						,   is_encoder_decoder=__A						,   tie_word_embeddings=__A						,   forced_eos_token_id=__A						,   max_length=__A						,   num_beams=__A						,   length_penalty=__A						,   early_stopping=__A						,   **__A						,   )



							def 					__A       (     self:      Optional[int]      )			-> List[Any]:
												_A									=       copy.deepcopy(self.__dict__      )
												_A									=       self.decoder.to_dict()
												_A									=       self.__class__.model_type
												return output



