


import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case            =   16
__snake_case            =   32






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		Any       ,  SCREAMING_SNAKE_CASE_   :		List[str] = 16	):





   """simple docstring"""




   UpperCamelCase         =					AutoTokenizer.from_pretrained("""bert-base-cased"""	)
   UpperCamelCase         =					load_dataset("""glue"""       ,  """mrpc"""	)

   def tokenize_function(SCREAMING_SNAKE_CASE_   :		Union[str, Any]	):
      # max_length=None => use the model max length (it's actually the default)
      UpperCamelCase         =					tokenizer(examples["""sentence1"""]       ,  examples["""sentence2"""]       ,  truncation=lowercase_       ,  max_length=lowercase_	)
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      UpperCamelCase         =					datasets.map(
          lowercase_       ,  batched=lowercase_       ,  remove_columns=["""idx""", """sentence1""", """sentence2"""]       ,  )

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   UpperCamelCase         =					tokenized_datasets.rename_column("""label"""       ,  """labels"""	)

   def collate_fn(SCREAMING_SNAKE_CASE_   :		int	):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      UpperCamelCase         =					128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         UpperCamelCase         =					16
      elif accelerator.mixed_precision != "no":
         UpperCamelCase         =					8
      else:
         UpperCamelCase         =					None

      return tokenizer.pad(
          lowercase_       ,  padding="""longest"""       ,  max_length=lowercase_       ,  pad_to_multiple_of=lowercase_       ,  return_tensors="""pt"""       ,  )

   # Instantiate dataloaders.
   UpperCamelCase         =					DataLoader(
       tokenized_datasets["""train"""]       ,  shuffle=lowercase_       ,  collate_fn=lowercase_       ,  batch_size=lowercase_	)
   UpperCamelCase         =					DataLoader(
       tokenized_datasets["""validation"""]       ,  shuffle=lowercase_       ,  collate_fn=lowercase_       ,  batch_size=lowercase_	)

   return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
   from accelerate.test_utils.training import mocked_dataloaders

   __snake_case            =   mocked_dataloaders  # noqa: F811






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		int       ,  SCREAMING_SNAKE_CASE_   :		Tuple	):





   """simple docstring"""




   if os.environ.get("""TESTING_MOCKED_DATALOADERS"""       ,  lowercase_	) == "1":
      UpperCamelCase         =					2
   # Initialize accelerator
   UpperCamelCase         =					Accelerator(cpu=args.cpu       ,  mixed_precision=args.mixed_precision	)
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   UpperCamelCase         =					config["lr"]
   UpperCamelCase         =					int(config["""num_epochs"""]	)
   UpperCamelCase         =					int(config["""seed"""]	)
   UpperCamelCase         =					int(config["""batch_size"""]	)

   UpperCamelCase         =					evaluate.load("""glue"""       ,  """mrpc"""	)

   # If the batch size is too big we use gradient accumulation
   UpperCamelCase         =					1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      UpperCamelCase         =					batch_size // MAX_GPU_BATCH_SIZE
      UpperCamelCase         =					MAX_GPU_BATCH_SIZE

   set_seed(lowercase_	)
   UpperCamelCase         =					get_dataloaders(lowercase_       ,  lowercase_	)
   # Instantiate the model (we build the model here so that the seed also control new weights initialization)
   UpperCamelCase         =					AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""       ,  return_dict=lowercase_	)

   # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
   # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
   # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
   UpperCamelCase         =					model.to(accelerator.device	)

   # Instantiate optimizer
   UpperCamelCase         =					AdamW(params=model.parameters()       ,  lr=lowercase_	)

   # Instantiate scheduler
   UpperCamelCase         =					get_linear_schedule_with_warmup(
       optimizer=lowercase_       ,  num_warmup_steps=100       ,  num_training_steps=(len(lowercase_	) * num_epochs) // gradient_accumulation_steps       ,  )

   # Prepare everything
   # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
   # prepare method.
   UpperCamelCase         =					accelerator.prepare(
       lowercase_       ,  lowercase_       ,  lowercase_       ,  lowercase_       ,  lowercase_	)

   # Now we train the model
   for epoch in range(lowercase_	):
      model.train()
      for step, batch in enumerate(lowercase_	):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device	)
         UpperCamelCase         =					model(**lowercase_	)
         UpperCamelCase         =					outputs.loss
         UpperCamelCase         =					loss / gradient_accumulation_steps
         accelerator.backward(lowercase_	)
         if step % gradient_accumulation_steps == 0:
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()

      model.eval()
      UpperCamelCase         =					0
      for step, batch in enumerate(lowercase_	):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device	)
         with torch.no_grad():
            UpperCamelCase         =					model(**lowercase_	)
         UpperCamelCase         =					outputs.logits.argmax(dim=-1	)
         UpperCamelCase         =					accelerator.gather((predictions, batch["""labels"""])	)
         # New Code #
         # First we check if it's a distributed system
         if accelerator.use_distributed:
            # Then see if we're on the last batch of our eval dataloader
            if step == len(lowercase_	) - 1:
               # Last batch needs to be truncated on distributed systems as it contains additional samples
               UpperCamelCase         =					predictions[: len(eval_dataloader.dataset	) - samples_seen]
               UpperCamelCase         =					references[: len(eval_dataloader.dataset	) - samples_seen]
            else:
               # Otherwise we add the number of samples seen
               samples_seen += references.shape[0]
            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
            # accelerator.gather_for_metrics((predictions, batch["labels"]))
         metric.add_batch(
             predictions=lowercase_       ,  references=lowercase_       ,  )

      UpperCamelCase         =					metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(f'epoch {epoch}:'       ,  lowercase_	)






def 							_lowercase    (		):





   """simple docstring"""




   UpperCamelCase         =					argparse.ArgumentParser(description="""Simple example of training script."""	)
   parser.add_argument(
       """--mixed_precision"""       ,  type=lowercase_       ,  default=lowercase_       ,  choices=["""no""", """fp16""", """bf16""", """fp8"""]       ,  help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""       ,  )
   parser.add_argument("""--cpu"""       ,  action="""store_true"""       ,  help="""If passed, will train on the CPU."""	)
   UpperCamelCase         =					parser.parse_args()
   UpperCamelCase         =					{"lr": 2e-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
   training_function(lowercase_       ,  lowercase_	)


if __name__ == "__main__":
   main()








import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__snake_case            =   logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class    UpperCAmelCase					(       __snake_case    ):




			def __init__(						self							:		Dict       ,       *__magic_name__							:		Union[str, Any]       ,       __magic_name__							:		Any=None       ,       __magic_name__							:		Union[str, Any]=None       ,       __magic_name__							:		Any=None       ,       **__magic_name__							:		Optional[Any]						):







						"""simple docstring"""







						super().__init__(*__magic_name__       ,       **__magic_name__						)
						UpperCamelCase         =					eval_examples
						UpperCamelCase         =					post_process_function
						UpperCamelCase         =					quant_trainer_args
						UpperCamelCase         =					1_2_8  # default number of calibration samples




			def   lowerCamelCase_					(						self							:		str       ,       __magic_name__							:		List[Any]=None						):







						"""simple docstring"""







						if calib_dataset is None and self.calib_dataset is None:
									raise ValueError("""Trainer: calibration requires an calib_dataset."""						)
						UpperCamelCase         =					calib_dataset if calib_dataset is not None else self.calib_dataset

						UpperCamelCase         =					self._remove_unused_columns(__magic_name__       ,       description="""Calibration"""						)

						return DataLoader(
						    __magic_name__       ,       batch_size=self.args.eval_batch_size       ,       collate_fn=self.data_collator       ,       drop_last=self.args.dataloader_drop_last       ,       num_workers=self.args.dataloader_num_workers       ,       pin_memory=self.args.dataloader_pin_memory       ,       shuffle=__magic_name__       ,       )




			def   lowerCamelCase_					(						self							:		Tuple       ,       __magic_name__							:		List[str]=None						):







						"""simple docstring"""







						UpperCamelCase         =					self.train_dataset if calib_dataset is None else calib_dataset
						UpperCamelCase         =					self.get_calib_dataloader(__magic_name__						)

						UpperCamelCase         =					self.model
						quant_trainer.configure_model(__magic_name__       ,       self.quant_trainer_args       ,       calib=__magic_name__						)
						model.eval()
						quant_trainer.enable_calibration(__magic_name__						)

						logger.info("""***** Running calibration *****"""						)
						logger.info(F'  Num examples = {self.calib_num}'						)
						logger.info(F'  Batch size = {calib_dataloader.batch_size}'						)

						for step, inputs in enumerate(__magic_name__						):
									# Prediction step
									UpperCamelCase  ,  UpperCamelCase  ,  UpperCamelCase         =					self.prediction_step(__magic_name__       ,       __magic_name__       ,       prediction_loss_only=__magic_name__						)
									if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
												break

						quant_trainer.finish_calibration(__magic_name__       ,       self.quant_trainer_args						)
						UpperCamelCase         =					model




			def   lowerCamelCase_					(						self							:		Any       ,       __magic_name__							:		Optional[int]=None       ,       __magic_name__							:		Optional[Any]=None       ,       __magic_name__							:		Union[str, Any]=None       ,       __magic_name__							:		str = "eval"						):







						"""simple docstring"""







						UpperCamelCase         =					self.eval_dataset if eval_dataset is None else eval_dataset
						UpperCamelCase         =					self.get_eval_dataloader(__magic_name__						)
						UpperCamelCase         =					self.eval_examples if eval_examples is None else eval_examples

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCamelCase         =					self.compute_metrics
						UpperCamelCase         =					None
						UpperCamelCase         =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						try:
									UpperCamelCase         =					eval_loop(
									    __magic_name__       ,       description="""Evaluation"""       ,       prediction_loss_only=True if compute_metrics is None else None       ,       ignore_keys=__magic_name__       ,       )
						finally:
									UpperCamelCase         =					compute_metrics

						if self.post_process_function is not None and self.compute_metrics is not None:
									UpperCamelCase         =					self.post_process_function(__magic_name__       ,       __magic_name__       ,       output.predictions						)
									UpperCamelCase         =					self.compute_metrics(__magic_name__						)

									# Prefix all keys with metric_key_prefix + '_'
									for key in list(metrics.keys()						):
												if not key.startswith(F'{metric_key_prefix}_'						):
															UpperCamelCase         =					metrics.pop(__magic_name__						)

									self.log(__magic_name__						)
						else:
									UpperCamelCase         =					{}

						if self.args.tpu_metrics_debug or self.args.debug:
									# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
									xm.master_print(met.metrics_report()						)

						UpperCamelCase         =					self.callback_handler.on_evaluate(self.args       ,       self.state       ,       self.control       ,       __magic_name__						)
						return metrics




			def   lowerCamelCase_					(						self							:		Dict       ,       __magic_name__							:		List[str]       ,       __magic_name__							:		List[Any]       ,       __magic_name__							:		List[Any]=None       ,       __magic_name__							:		str = "test"						):







						"""simple docstring"""







						UpperCamelCase         =					self.get_test_dataloader(__magic_name__						)

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCamelCase         =					self.compute_metrics
						UpperCamelCase         =					None
						UpperCamelCase         =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						try:
									UpperCamelCase         =					eval_loop(
									    __magic_name__       ,       description="""Prediction"""       ,       prediction_loss_only=True if compute_metrics is None else None       ,       ignore_keys=__magic_name__       ,       )
						finally:
									UpperCamelCase         =					compute_metrics

						if self.post_process_function is None or self.compute_metrics is None:
									return output

						UpperCamelCase         =					self.post_process_function(__magic_name__       ,       __magic_name__       ,       output.predictions       ,       """predict"""						)
						UpperCamelCase         =					self.compute_metrics(__magic_name__						)

						# Prefix all keys with metric_key_prefix + '_'
						for key in list(metrics.keys()						):
									if not key.startswith(F'{metric_key_prefix}_'						):
												UpperCamelCase         =					metrics.pop(__magic_name__						)

						return PredictionOutput(predictions=predictions.predictions       ,       label_ids=predictions.label_ids       ,       metrics=__magic_name__						)





			def   lowerCamelCase_					(						self							:		List[str]       ,       __magic_name__							:		Optional[Any]="./"						):







						"""simple docstring"""







						UpperCamelCase         =					self.eval_dataset
						UpperCamelCase         =					self.get_eval_dataloader(__magic_name__						)

						UpperCamelCase         =					next(iter(__magic_name__						)						)

						# saving device - to make it consistent
						UpperCamelCase         =					torch.device("""cuda""" if torch.cuda.is_available() else """cpu"""						)

						# convert to tuple
						UpperCamelCase         =					tuple(v.to(__magic_name__						) for k, v in batch.items()						)

						logger.info("""Converting model to be onnx compatible"""						)
						from pytorch_quantization.nn import TensorQuantizer

						UpperCamelCase         =					True

						UpperCamelCase         =					self.model.to(__magic_name__						)

						model.eval()
						model.float()

						UpperCamelCase         =					model.module if hasattr(__magic_name__       ,       """module"""						) else model
						quant_trainer.configure_model(__magic_name__       ,       self.quant_trainer_args						)

						UpperCamelCase         =					os.path.join(__magic_name__       ,       """model.onnx"""						)
						logger.info(F'exporting model to {output_model_file}'						)

						UpperCamelCase         =					{0: """batch_size""", 1: """seq_len"""}

						torch.onnx.export(
						    __magic_name__       ,       __magic_name__       ,       __magic_name__       ,       export_params=__magic_name__       ,       opset_version=1_3       ,       do_constant_folding=__magic_name__       ,       input_names=["""input_ids""", """attention_mask""", """token_type_ids"""]       ,       output_names=["""output_start_logits""", """output_end_logits"""]       ,       dynamic_axes={
						        """input_ids""": axes,
						        """attention_mask""": axes,
						        """token_type_ids""": axes,
						        """output_start_logits""": axes,
						        """output_end_logits""": axes,
						    }       ,       verbose=__magic_name__       ,       )
						logger.info("""onnx export finished"""						)




