






"""simple docstring"""





import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

UpperCamelCase =			32






def 				A(  snake_case_						,   snake_case_ = 16		):
       """simple docstring"""





       lowercase__:				Optional[int]			  =			AutoTokenizer.from_pretrained("bert-base-cased"		)
       lowercase__:				str			  =			load_dataset("glue"						,   "mrpc"		)

       def tokenize_function(snake_case_		):
              # max_length=None => use the model max length (it's actually the default)
              lowercase__:				str			  =			tokenizer(examples["sentence1"]						,   examples["sentence2"]						,   truncation=lowerCAmelCase_						,   max_length=lowerCAmelCase_		)
              return outputs

       # Apply the method we just defined to all the examples in all the splits of the dataset
       # starting with the main process first:
       with accelerator.main_process_first():
              lowercase__:				List[str]			  =			datasets.map(
                  lowerCAmelCase_						,   batched=lowerCAmelCase_						,   remove_columns=["idx", "sentence1", "sentence2"]						,   )

       # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
       # transformers library
       lowercase__:				List[str]			  =			tokenized_datasets.rename_column("label"						,   "labels"		)

       def collate_fn(snake_case_		):
              # When using mixed precision we want round multiples of 8/16
              if accelerator.mixed_precision == "fp8":
                     lowercase__:				str			  =			16
              elif accelerator.mixed_precision != "no":
                     lowercase__:				Tuple			  =			8
              else:
                     lowercase__:				Dict			  =			None

              return tokenizer.pad(
                  lowerCAmelCase_						,   padding="longest"						,   pad_to_multiple_of=lowerCAmelCase_						,   return_tensors="pt"						,   )

       # Instantiate dataloaders.
       lowercase__:				List[Any]			  =			DataLoader(
           tokenized_datasets["train"]						,   shuffle=lowerCAmelCase_						,   collate_fn=lowerCAmelCase_						,   batch_size=lowerCAmelCase_		)
       lowercase__:				List[Any]			  =			DataLoader(
           tokenized_datasets["validation"]						,   shuffle=lowerCAmelCase_						,   collate_fn=lowerCAmelCase_						,   batch_size=lowerCAmelCase_		)

       return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  UpperCamelCase =			mocked_dataloaders  # noqa: F811






def 				A(  snake_case_						,   snake_case_		):
       """simple docstring"""





       if os.environ.get("TESTING_MOCKED_DATALOADERS"						,   lowerCAmelCase_		) == "1":
              lowercase__:				List[str]			  =			2
       # Initialize accelerator
       lowercase__:				str			  =			Accelerator(cpu=args.cpu						,   mixed_precision=args.mixed_precision		)
       # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
       lowercase__:				Union[str, Any]			  =			config["""lr"""]
       lowercase__:				Any			  =			int(config["num_epochs"]		)
       lowercase__:				Union[str, Any]			  =			int(config["seed"]		)
       lowercase__:				List[str]			  =			int(config["batch_size"]		)

       lowercase__:				str			  =			evaluate.load("glue"						,   "mrpc"		)

       # New Code #
       # We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
       # to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
       # half each time. From this, we can calculate the number of gradient accumulation steps needed
       # and modify the Accelerator object as a result
       @find_executable_batch_size(starting_batch_size=int(lowerCAmelCase_		)		)
       def inner_training_loop(snake_case_		):
              # Since we need to modify the outside accelerator object, we need to bring it
              # to the local scope
              nonlocal accelerator

              # We can calculate the number of gradient accumulation steps based on the current
              # batch size vs the starting batch size
              lowercase__:				Optional[Any]			  =			observed_batch_size // batch_size

              # And then set it in the Accelerator directly:
              lowercase__:				Union[str, Any]			  =			num_gradient_accumulation_steps

              # Next we need to free all of the stored model references in the Accelerator each time
              accelerator.free_memory()

              # And set the seed so our results are reproducable each reset
              set_seed(lowerCAmelCase_		)

              # Instantiate the model (we build the model here so that the seed also control new weights initialization)
              lowercase__:				Dict			  =			AutoModelForSequenceClassification.from_pretrained("bert-base-cased"						,   return_dict=lowerCAmelCase_		)

              # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
              # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
              # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
              lowercase__:				Optional[int]			  =			model.to(accelerator.device		)

              # Instantiate optimizer
              lowercase__:				Union[str, Any]			  =			AdamW(params=model.parameters()						,   lr=lowerCAmelCase_		)
              lowercase__:				int			  =			get_dataloaders(lowerCAmelCase_						,   lowerCAmelCase_		)

              # Instantiate scheduler
              lowercase__:				int			  =			get_linear_schedule_with_warmup(
                  optimizer=lowerCAmelCase_						,   num_warmup_steps=100						,   num_training_steps=(len(lowerCAmelCase_		) * num_epochs)						,   )

              # Prepare everything
              # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
              # prepare method.
              lowercase__:				int			  =			accelerator.prepare(
                  lowerCAmelCase_						,   lowerCAmelCase_						,   lowerCAmelCase_						,   lowerCAmelCase_						,   lowerCAmelCase_		)

              # Now we train the model
              for epoch in range(lowerCAmelCase_		):
                     model.train()
                     for step, batch in enumerate(lowerCAmelCase_		):
                            # And perform gradient accumulation
                            with accelerator.accumulate(lowerCAmelCase_		):
                                   # We could avoid this line since we set the accelerator with `device_placement=True`.
                                   batch.to(accelerator.device		)
                                   lowercase__:				Any			  =			model(**lowerCAmelCase_		)
                                   lowercase__:				Optional[int]			  =			outputs.loss
                                   accelerator.backward(lowerCAmelCase_		)
                                   optimizer.step()
                                   lr_scheduler.step()
                                   optimizer.zero_grad()

                     model.eval()
                     for step, batch in enumerate(lowerCAmelCase_		):
                            # We could avoid this line since we set the accelerator with `device_placement=True`.
                            batch.to(accelerator.device		)
                            with torch.no_grad():
                                   lowercase__:				Dict			  =			model(**lowerCAmelCase_		)
                            lowercase__:				Any			  =			outputs.logits.argmax(dim=-1		)
                            lowercase__:				Dict			  =			accelerator.gather_for_metrics((predictions, batch["labels"])		)
                            metric.add_batch(
                                predictions=lowerCAmelCase_						,   references=lowerCAmelCase_						,   )

                     lowercase__:				List[str]			  =			metric.compute()
                     # Use accelerator.print to print only on the main process.
                     accelerator.print(F"""epoch {epoch}:"""						,   lowerCAmelCase_		)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
       inner_training_loop()






def 				A(  ):
       """simple docstring"""





       lowercase__:				Optional[int]			  =			argparse.ArgumentParser(description="Simple example of training script."		)
       parser.add_argument(
           "--mixed_precision"						,   type=lowerCAmelCase_						,   default=lowerCAmelCase_						,   choices=["no", "fp16", "bf16", "fp8"]						,   help="Whether to use mixed precision. Choose"
           "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
           "and an Nvidia Ampere GPU."						,   )
       parser.add_argument("--cpu"						,   action="store_true"						,   help="If passed, will train on the CPU."		)
       lowercase__:				List[Any]			  =			parser.parse_args()
       # New Code #
       # We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
       lowercase__:				List[Any]			  =			{"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 256}
       training_function(lowerCAmelCase_						,   lowerCAmelCase_		)


if __name__ == "__main__":
  main()








'''simple docstring'''

import unittest

from transformers import is_torch_available, load_tool

from .test_tools_common import ToolTesterMixin


if is_torch_available():
   import torch







class 	__lowerCAmelCase    (  unittest.TestCase       ,					__a				):



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Tuple         =    load_tool("""speech-to-text"""	)
          self.tool.setup()



     def     snake_case_ (self	):
          _UpperCAmelCase					:  str         =    self.tool(torch.ones(3_0_0_0	)	)
          self.assertEqual(lowerCAmelCase__						,	""" you"""	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Union[str, Any]         =    self.tool(audio=torch.ones(3_0_0_0	)	)
          self.assertEqual(lowerCAmelCase__						,	""" you"""	)


