
import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


snake_case__					=				16
snake_case__					=				32

def 							lowerCamelCase_   (      UpperCAmelCase_				:					List[str]				,      UpperCAmelCase_				:					Union[str, Any] = 16							):
						lowercase							:     Any									=	AutoTokenizer.from_pretrained('''bert-base-cased'''							)
						lowercase							:     int									=	load_dataset('''glue'''				,      '''mrpc'''							)

						def tokenize_function(UpperCAmelCase_				:					Tuple							):
												# max_length=None => use the model max length (it's actually the default)
												lowercase							:     Dict									=	tokenizer(examples['''sentence1''']				,      examples['''sentence2''']				,      truncation=lowerCAmelCase_				,      max_length=lowerCAmelCase_							)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												lowercase							:     Tuple									=	datasets.map(
												    lowerCAmelCase_				,      batched=lowerCAmelCase_				,      remove_columns=['''idx''', '''sentence1''', '''sentence2''']				,      )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						lowercase							:     str									=	tokenized_datasets.rename_column('''label'''				,      '''labels'''							)

						def collate_fn(UpperCAmelCase_				:					str							):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												lowercase							:     int									=	1_28 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		lowercase							:     Tuple									=	16
												elif accelerator.mixed_precision != "no":
																		lowercase							:     str									=	8
												else:
																		lowercase							:     Tuple									=	None

												return tokenizer.pad(
												    lowerCAmelCase_				,      padding='''longest'''				,      max_length=lowerCAmelCase_				,      pad_to_multiple_of=lowerCAmelCase_				,      return_tensors='''pt'''				,      )

						# Instantiate dataloaders.
						lowercase							:     List[str]									=	DataLoader(
						    tokenized_datasets['''train''']				,      shuffle=lowerCAmelCase_				,      collate_fn=lowerCAmelCase_				,      batch_size=lowerCAmelCase_							)
						lowercase							:     str									=	DataLoader(
						    tokenized_datasets['''validation''']				,      shuffle=lowerCAmelCase_				,      collate_fn=lowerCAmelCase_				,      batch_size=lowerCAmelCase_							)

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
				from accelerate.test_utils.training import mocked_dataloaders

				snake_case__					=				mocked_dataloaders  # noqa: F811

def 							lowerCamelCase_   (      UpperCAmelCase_				:					int				,      UpperCAmelCase_				:					Any							):
						# For testing only
						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''				,      lowerCAmelCase_							) == "1":
												lowercase							:     Optional[Any]									=	2
						# New Code #
						lowercase							:     List[Any]									=	int(args.gradient_accumulation_steps							)
						lowercase							:     str									=	int(args.local_sgd_steps							)
						# Initialize accelerator
						lowercase							:     Union[str, Any]									=	Accelerator(
						    cpu=args.cpu				,      mixed_precision=args.mixed_precision				,      gradient_accumulation_steps=lowerCAmelCase_							)
						if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
												raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''							)
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						lowercase							:     int									=	config['lr']
						lowercase							:     str									=	int(config['''num_epochs''']							)
						lowercase							:     List[Any]									=	int(config['''seed''']							)
						lowercase							:     Optional[Any]									=	int(config['''batch_size''']							)

						lowercase							:     Any									=	evaluate.load('''glue'''				,      '''mrpc'''							)

						set_seed(lowerCAmelCase_							)
						lowercase							:     Union[str, Any]									=	get_dataloaders(lowerCAmelCase_				,      lowerCAmelCase_							)
						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						lowercase							:     List[Any]									=	AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''				,      return_dict=lowerCAmelCase_							)

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						lowercase							:     Tuple									=	model.to(accelerator.device							)

						# Instantiate optimizer
						lowercase							:     List[Any]									=	AdamW(params=model.parameters()				,      lr=lowerCAmelCase_							)

						# Instantiate scheduler
						lowercase							:     Union[str, Any]									=	get_linear_schedule_with_warmup(
						    optimizer=lowerCAmelCase_				,      num_warmup_steps=1_00				,      num_training_steps=(len(lowerCAmelCase_							) * num_epochs)				,      )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						lowercase							:     Any									=	accelerator.prepare(
						    lowerCAmelCase_				,      lowerCAmelCase_				,      lowerCAmelCase_				,      lowerCAmelCase_				,      lowerCAmelCase_							)

						# Now we train the model
						for epoch in range(lowerCAmelCase_							):
												model.train()
												with LocalSGD(
												    accelerator=lowerCAmelCase_				,      model=lowerCAmelCase_				,      local_sgd_steps=lowerCAmelCase_				,      enabled=local_sgd_steps is not None							) as local_sgd:
																		for step, batch in enumerate(lowerCAmelCase_							):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device							)
																								# New code #
																								# We use the new `accumulate` context manager to perform gradient accumulation
																								# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																								with accelerator.accumulate(lowerCAmelCase_							):
																														lowercase							:     Dict									=	model(**lowerCAmelCase_							)
																														lowercase							:     Any									=	output.loss
																														accelerator.backward(lowerCAmelCase_							)
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()
																														# LocalSGD-specific line
																														local_sgd.step()

												model.eval()
												for step, batch in enumerate(lowerCAmelCase_							):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device							)
																		with torch.no_grad():
																								lowercase							:     Any									=	model(**lowerCAmelCase_							)
																		lowercase							:     Any									=	outputs.logits.argmax(dim=-1							)
																		lowercase							:     Dict									=	accelerator.gather_for_metrics((predictions, batch['''labels'''])							)
																		metric.add_batch(
																		    predictions=lowerCAmelCase_				,      references=lowerCAmelCase_				,      )

												lowercase							:     List[Any]									=	metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f'''epoch {epoch}:'''				,      lowerCAmelCase_							)

def 							lowerCamelCase_   (      ):
						lowercase							:     List[str]									=	argparse.ArgumentParser(description='''Simple example of training script.'''							)
						parser.add_argument(
						    '''--mixed_precision'''				,      type=lowerCAmelCase_				,      default=lowerCAmelCase_				,      choices=['''no''', '''fp16''', '''bf16''', '''fp8''']				,      help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''				,      )
						# New Code #
						parser.add_argument(
						    '''--gradient_accumulation_steps'''				,      type=lowerCAmelCase_				,      default=1				,      help='''The number of minibatches to be ran before gradients are accumulated.'''				,      )
						parser.add_argument(
						    '''--local_sgd_steps'''				,      type=lowerCAmelCase_				,      default=8				,      help='''Number of local SGD steps or None to disable local SGD'''							)
						parser.add_argument('''--cpu'''				,      action='''store_true'''				,      help='''If passed, will train on the CPU.'''							)
						lowercase							:     Tuple									=	parser.parse_args()
						lowercase							:     int									=	{'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
						training_function(lowerCAmelCase_				,      lowerCAmelCase_							)


if __name__ == "__main__":
				main()







'''simple docstring'''

import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()



def 	__lowerCamelCase	( lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_    )     -> Any:
		# Initialise PyTorch model
		_a	: int								=				BigBirdConfig.from_json_file(lowerCAmelCase_    )
		print(f"""Building PyTorch model from configuration: {config}"""    )

		if is_trivia_qa:
				_a	: Union[str, Any]								=				BigBirdForQuestionAnswering(lowerCAmelCase_    )
		else:
				_a	: Any								=				BigBirdForPreTraining(lowerCAmelCase_    )

		# Load weights from tf checkpoint
		load_tf_weights_in_big_bird(lowerCAmelCase_		,     lowerCAmelCase_		,     is_trivia_qa=lowerCAmelCase_    )

		# Save pytorch-model
		print(f"""Save PyTorch model to {pytorch_dump_path}"""    )
		model.save_pretrained(lowerCAmelCase_    )


if __name__ == "__main__":
							__lowerCAmelCase		     =  argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
							)
							parser.add_argument(
							    '''--big_bird_config_file''',
							    default=None,
							    type=str,
							    required=True,
							    help=(
							        '''The config json file corresponding to the pre-trained BERT model. \n'''
							        '''This specifies the model architecture.'''
							    ),
							)
							parser.add_argument(
							    '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
							)
							parser.add_argument(
							    '''--is_trivia_qa''', action='''store_true''', help='''Whether to convert a model with a trivia_qa head.'''
							)
							__lowerCAmelCase		     =  parser.parse_args()
							convert_tf_checkpoint_to_pytorch(
							    args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
							)




