






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a__       =     1_6
a__       =     3_2

def _UpperCAmelCase   (	a						:	Accelerator  ,					a						:	int = 16 ):
							snake_case__	  =       AutoTokenizer.from_pretrained("""bert-base-cased""" )
							snake_case__	  =       load_dataset("""glue"""  ,					"""mrpc""" )

							def tokenize_function(a						:	Optional[Any] ):
														# max_length=None => use the model max length (it's actually the default)
														snake_case__	  =       tokenizer(examples["""sentence1"""]  ,					examples["""sentence2"""]  ,					truncation=a  ,					max_length=a )
														return outputs

							# Apply the method we just defined to all the examples in all the splits of the dataset
							# starting with the main process first:
							with accelerator.main_process_first():
														snake_case__	  =       datasets.map(
														    a  ,					batched=a  ,					remove_columns=["""idx""", """sentence1""", """sentence2"""]  ,					)

							# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
							# transformers library
							snake_case__	  =       tokenized_datasets.rename_column("""label"""  ,					"""labels""" )

							def collate_fn(a						:	List[str] ):
														# On TPU it's best to pad everything to the same length or training will be very slow.
														snake_case__	  =       128 if accelerator.distributed_type == DistributedType.TPU else None
														# When using mixed precision we want round multiples of 8/16
														if accelerator.mixed_precision == "fp8":
																					snake_case__	  =       16
														elif accelerator.mixed_precision != "no":
																					snake_case__	  =       8
														else:
																					snake_case__	  =       None

														return tokenizer.pad(
														    a  ,					padding="""longest"""  ,					max_length=a  ,					pad_to_multiple_of=a  ,					return_tensors="""pt"""  ,					)

							# Instantiate dataloaders.
							snake_case__	  =       DataLoader(
							    tokenized_datasets["""train"""]  ,					shuffle=a  ,					collate_fn=a  ,					batch_size=a )
							snake_case__	  =       DataLoader(
							    tokenized_datasets["""validation"""]  ,					shuffle=a  ,					collate_fn=a  ,					batch_size=a )

							return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
	from accelerate.test_utils.training import mocked_dataloaders

	a__       =     mocked_dataloaders  # noqa: F811

def _UpperCAmelCase   (	a						:	Union[str, Any]  ,					a						:	Dict ):
							# For testing only
							if os.environ.get("""TESTING_MOCKED_DATALOADERS"""  ,					a ) == "1":
														snake_case__	  =       2
							# New Code #
							snake_case__	  =       int(args.gradient_accumulation_steps )
							snake_case__	  =       int(args.local_sgd_steps )
							# Initialize accelerator
							snake_case__	  =       Accelerator(
							    cpu=args.cpu  ,					mixed_precision=args.mixed_precision  ,					gradient_accumulation_steps=a )
							if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
														raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)""" )
							# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
							snake_case__	  =       config["""lr"""]
							snake_case__	  =       int(config["""num_epochs"""] )
							snake_case__	  =       int(config["""seed"""] )
							snake_case__	  =       int(config["""batch_size"""] )

							snake_case__	  =       evaluate.load("""glue"""  ,					"""mrpc""" )

							set_seed(a )
							snake_case__  , snake_case__	  =       get_dataloaders(a  ,					a )
							# Instantiate the model (we build the model here so that the seed also control new weights initialization)
							snake_case__	  =       AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""  ,					return_dict=a )

							# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
							# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
							# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
							snake_case__	  =       model.to(accelerator.device )

							# Instantiate optimizer
							snake_case__	  =       AdamW(params=model.parameters()  ,					lr=a )

							# Instantiate scheduler
							snake_case__	  =       get_linear_schedule_with_warmup(
							    optimizer=a  ,					num_warmup_steps=100  ,					num_training_steps=(len(a ) * num_epochs)  ,					)

							# Prepare everything
							# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
							# prepare method.
							snake_case__  , snake_case__  , snake_case__  , snake_case__  , snake_case__	  =       accelerator.prepare(
							    a  ,					a  ,					a  ,					a  ,					a )

							# Now we train the model
							for epoch in range(a ):
														model.train()
														with LocalSGD(
														    accelerator=a  ,					model=a  ,					local_sgd_steps=a  ,					enabled=local_sgd_steps is not None ) as local_sgd:
																					for step, batch in enumerate(a ):
																												# We could avoid this line since we set the accelerator with `device_placement=True`.
																												batch.to(accelerator.device )
																												# New code #
																												# We use the new `accumulate` context manager to perform gradient accumulation
																												# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																												with accelerator.accumulate(a ):
																																			snake_case__	  =       model(**a )
																																			snake_case__	  =       output.loss
																																			accelerator.backward(a )
																																			optimizer.step()
																																			lr_scheduler.step()
																																			optimizer.zero_grad()
																																			# LocalSGD-specific line
																																			local_sgd.step()

														model.eval()
														for step, batch in enumerate(a ):
																					# We could avoid this line since we set the accelerator with `device_placement=True`.
																					batch.to(accelerator.device )
																					with torch.no_grad():
																												snake_case__	  =       model(**a )
																					snake_case__	  =       outputs.logits.argmax(dim=-1 )
																					snake_case__  , snake_case__	  =       accelerator.gather_for_metrics((predictions, batch["""labels"""]) )
																					metric.add_batch(
																					    predictions=a  ,					references=a  ,					)

														snake_case__	  =       metric.compute()
														# Use accelerator.print to print only on the main process.
														accelerator.print(F'''epoch {epoch}:'''  ,					a )

def _UpperCAmelCase   (	):
							snake_case__	  =       argparse.ArgumentParser(description="""Simple example of training script.""" )
							parser.add_argument(
							    """--mixed_precision"""  ,					type=a  ,					default=a  ,					choices=["""no""", """fp16""", """bf16""", """fp8"""]  ,					help="""Whether to use mixed precision. Choose"""
							    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
							    """and an Nvidia Ampere GPU."""  ,					)
							# New Code #
							parser.add_argument(
							    """--gradient_accumulation_steps"""  ,					type=a  ,					default=1  ,					help="""The number of minibatches to be ran before gradients are accumulated."""  ,					)
							parser.add_argument(
							    """--local_sgd_steps"""  ,					type=a  ,					default=8  ,					help="""Number of local SGD steps or None to disable local SGD""" )
							parser.add_argument("""--cpu"""  ,					action="""store_true"""  ,					help="""If passed, will train on the CPU.""" )
							snake_case__	  =       parser.parse_args()
							snake_case__	  =       {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
							training_function(a  ,					a )


if __name__ == "__main__":
	main()








from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_torch_available, is_torch_tpu_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_torch_available():
	import torch

if is_torch_tpu_available(check_device=False):
	import torch_xla.core.xla_model as xm


a__       =     logging.get_logger(__name__)


@dataclass
class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				_lowercase				:   Dict      =  [
				    '''no_inference''',
				    '''no_cuda''',
				    '''no_tpu''',
				    '''no_speed''',
				    '''no_memory''',
				    '''no_env_print''',
				    '''no_multi_process''',
				]



				def __init__(      self    :    str			,		**UpperCamelCase__    :    Dict):






											'''simple docstring'''







											for deprecated_arg in self.deprecated_args:
																		if deprecated_arg in kwargs:
																									snake_case__	  =       deprecated_arg[3:]
																									setattr(self			,		UpperCamelCase__			,		not kwargs.pop(UpperCamelCase__))
																									logger.warning(
																									    F'''{deprecated_arg} is depreciated. Please use --no_{positive_arg} or'''
																									    F''' {positive_arg}={kwargs[positive_arg]}''')

											snake_case__	  =       kwargs.pop("""torchscript"""			,		self.torchscript)
											snake_case__	  =       kwargs.pop("""torch_xla_tpu_print_metrics"""			,		self.torch_xla_tpu_print_metrics)
											snake_case__	  =       kwargs.pop("""fp16_opt_level"""			,		self.fpaa_opt_level)
											super().__init__(**UpperCamelCase__)

				_lowercase				:   bool      =  field(default=lowercase_     ,      metadata={'''help''': '''Trace the models using torchscript'''}      )
				_lowercase				:   bool      =  field(default=lowercase_     ,      metadata={'''help''': '''Print Xla/PyTorch tpu metrics'''}      )
				_lowercase				:   str      =  field(
				    default='''O1'''     ,      metadata={
				        '''help''': (
				            '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\']. '''
				            '''See details at https://nvidia.github.io/apex/amp.html'''
				        )
				    }     ,      )



				@cached_property
				def 					__magic_name__							(      self    :    Tuple):






											'''simple docstring'''







											requires_backends(self			,		["""torch"""])
											logger.info("""PyTorch: setting up devices""")
											if not self.cuda:
																		snake_case__	  =       torch.device("""cpu""")
																		snake_case__	  =       0
											elif is_torch_tpu_available():
																		snake_case__	  =       xm.xla_device()
																		snake_case__	  =       0
											else:
																		snake_case__	  =       torch.device("""cuda""" if torch.cuda.is_available() else """cpu""")
																		snake_case__	  =       torch.cuda.device_count()
											return device, n_gpu



				@property
				def 					__magic_name__							(      self    :    Union[str, Any]):






											'''simple docstring'''







											return is_torch_tpu_available() and self.tpu



				@property
				def 					__magic_name__							(      self    :    List[str]):






											'''simple docstring'''







											requires_backends(self			,		["""torch"""])
											# TODO(PVP): currently only single GPU is supported
											return torch.cuda.current_device()



				@property
				def 					__magic_name__							(      self    :    Union[str, Any]):






											'''simple docstring'''







											requires_backends(self			,		["""torch"""])
											return self._setup_devices[0]



				@property
				def 					__magic_name__							(      self    :    str):






											'''simple docstring'''







											requires_backends(self			,		["""torch"""])
											return self._setup_devices[1]



				@property
				def 					__magic_name__							(      self    :    str):






											'''simple docstring'''







											return self.n_gpu > 0
