






"""simple docstring"""







import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

_UpperCamelCase     			=  32

def _a (		_snake_case      ,       _snake_case = 16							):







      """simple docstring"""






      UpperCAmelCase							=       AutoTokenizer.from_pretrained("""bert-base-cased"""							)
      UpperCAmelCase							=       load_dataset("""glue"""      ,       """mrpc"""							)

      def tokenize_function(_snake_case							):
            # max_length=None => use the model max length (it's actually the default)
            UpperCAmelCase							=       tokenizer(examples["""sentence1"""]      ,       examples["""sentence2"""]      ,       truncation=_snake_case      ,       max_length=_snake_case							)
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            UpperCAmelCase							=       datasets.map(
                _snake_case      ,       batched=_snake_case      ,       remove_columns=["""idx""", """sentence1""", """sentence2"""]      ,       )

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      UpperCAmelCase							=       tokenized_datasets.rename_column("""label"""      ,       """labels"""							)

      def collate_fn(_snake_case							):
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  UpperCAmelCase							=       16
            elif accelerator.mixed_precision != "no":
                  UpperCAmelCase							=       8
            else:
                  UpperCAmelCase							=       None

            return tokenizer.pad(
                _snake_case      ,       padding="""longest"""      ,       pad_to_multiple_of=_snake_case      ,       return_tensors="""pt"""      ,       )

      # Instantiate dataloaders.
      UpperCAmelCase							=       DataLoader(
          tokenized_datasets["""train"""]      ,       shuffle=_snake_case      ,       collate_fn=_snake_case      ,       batch_size=_snake_case							)
      UpperCAmelCase							=       DataLoader(
          tokenized_datasets["""validation"""]      ,       shuffle=_snake_case      ,       collate_fn=_snake_case      ,       batch_size=_snake_case							)

      return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  _UpperCamelCase     			=  mocked_dataloaders  # noqa: F811

def _a (		_snake_case      ,       _snake_case							):







      """simple docstring"""






      if os.environ.get("""TESTING_MOCKED_DATALOADERS"""      ,       _snake_case							) == "1":
            UpperCAmelCase							=       2
      # Initialize accelerator
      UpperCAmelCase							=       Accelerator(cpu=args.cpu      ,       mixed_precision=args.mixed_precision							)
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      UpperCAmelCase							=       config["""lr"""]
      UpperCAmelCase							=       int(config["""num_epochs"""]							)
      UpperCAmelCase							=       int(config["""seed"""]							)
      UpperCAmelCase							=       int(config["""batch_size"""]							)

      UpperCAmelCase							=       evaluate.load("""glue"""      ,       """mrpc"""							)

      # New Code #
      # We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
      # to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
      # half each time. From this, we can calculate the number of gradient accumulation steps needed
      # and modify the Accelerator object as a result
      @find_executable_batch_size(starting_batch_size=int(_snake_case							)							)
      def inner_training_loop(_snake_case							):
            # Since we need to modify the outside accelerator object, we need to bring it
            # to the local scope
            nonlocal accelerator

            # We can calculate the number of gradient accumulation steps based on the current
            # batch size vs the starting batch size
            UpperCAmelCase							=       observed_batch_size // batch_size

            # And then set it in the Accelerator directly:
            UpperCAmelCase							=       num_gradient_accumulation_steps

            # Next we need to free all of the stored model references in the Accelerator each time
            accelerator.free_memory()

            # And set the seed so our results are reproducable each reset
            set_seed(_snake_case							)

            # Instantiate the model (we build the model here so that the seed also control new weights initialization)
            UpperCAmelCase							=       AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""      ,       return_dict=_snake_case							)

            # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
            # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
            # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
            UpperCAmelCase							=       model.to(accelerator.device							)

            # Instantiate optimizer
            UpperCAmelCase							=       AdamW(params=model.parameters()      ,       lr=_snake_case							)
            UpperCAmelCase			,					UpperCAmelCase							=       get_dataloaders(_snake_case      ,       _snake_case							)

            # Instantiate scheduler
            UpperCAmelCase							=       get_linear_schedule_with_warmup(
                optimizer=_snake_case      ,       num_warmup_steps=100      ,       num_training_steps=(len(_snake_case							) * num_epochs)      ,       )

            # Prepare everything
            # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
            # prepare method.
            UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase							=       accelerator.prepare(
                _snake_case      ,       _snake_case      ,       _snake_case      ,       _snake_case      ,       _snake_case							)

            # Now we train the model
            for epoch in range(_snake_case							):
                  model.train()
                  for step, batch in enumerate(_snake_case							):
                        # And perform gradient accumulation
                        with accelerator.accumulate(_snake_case							):
                              # We could avoid this line since we set the accelerator with `device_placement=True`.
                              batch.to(accelerator.device							)
                              UpperCAmelCase							=       model(**_snake_case							)
                              UpperCAmelCase							=       outputs.loss
                              accelerator.backward(_snake_case							)
                              optimizer.step()
                              lr_scheduler.step()
                              optimizer.zero_grad()

                  model.eval()
                  for step, batch in enumerate(_snake_case							):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device							)
                        with torch.no_grad():
                              UpperCAmelCase							=       model(**_snake_case							)
                        UpperCAmelCase							=       outputs.logits.argmax(dim=-1							)
                        UpperCAmelCase			,					UpperCAmelCase							=       accelerator.gather_for_metrics((predictions, batch["""labels"""])							)
                        metric.add_batch(
                            predictions=_snake_case      ,       references=_snake_case      ,       )

                  UpperCAmelCase							=       metric.compute()
                  # Use accelerator.print to print only on the main process.
                  accelerator.print(F'''epoch {epoch}:'''      ,       _snake_case							)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
      inner_training_loop()

def _a (		):







      """simple docstring"""






      UpperCAmelCase							=       argparse.ArgumentParser(description="""Simple example of training script."""							)
      parser.add_argument(
          """--mixed_precision"""      ,       type=_snake_case      ,       default=_snake_case      ,       choices=["""no""", """fp16""", """bf16""", """fp8"""]      ,       help="""Whether to use mixed precision. Choose"""
          """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
          """and an Nvidia Ampere GPU."""      ,       )
      parser.add_argument("""--cpu"""      ,       action="""store_true"""      ,       help="""If passed, will train on the CPU."""							)
      UpperCAmelCase							=       parser.parse_args()
      # New Code #
      # We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
      UpperCAmelCase							=       {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 256}
      training_function(_snake_case      ,       _snake_case							)


if __name__ == "__main__":
  main()











"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_UpperCamelCase     			=  logging.get_logger(__name__)







class 					lowerCamelCase__					(       snake_case ):
    SCREAMING_SNAKE_CASE		=			['''pixel_values''']






    def __init__( self	,A = True	,A = None	,A = PILImageResampling.BILINEAR	,A = True	,A = 1 / 255	,A = True	,A = None	,A = None	,**A	,):
          super().__init__(**A					)
          UpperCAmelCase							=       size if size is not None else {"""height""": 224, """width""": 224}
          UpperCAmelCase							=       get_size_dict(A					)
          UpperCAmelCase							=       do_resize
          UpperCAmelCase							=       do_rescale
          UpperCAmelCase							=       do_normalize
          UpperCAmelCase							=       size
          UpperCAmelCase							=       resample
          UpperCAmelCase							=       rescale_factor
          UpperCAmelCase							=       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
          UpperCAmelCase							=       image_std if image_std is not None else IMAGENET_STANDARD_STD






    def 				_UpperCamelCase							( self	,A	,A	,A = PILImageResampling.BILINEAR	,A = None	,**A	,):
          UpperCAmelCase							=       get_size_dict(A					)
          if "height" not in size or "width" not in size:
                raise ValueError(F'''The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}'''					)
          return resize(
              A	,size=(size["""height"""], size["""width"""])	,resample=A	,data_format=A	,**A					)






    def 				_UpperCamelCase							( self	,A	,A	,A = None	,**A					):
          return rescale(A	,scale=A	,data_format=A	,**A					)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A = None	,**A	,):
          return normalize(A	,mean=A	,std=A	,data_format=A	,**A					)






    def 				_UpperCamelCase							( self	,A	,A = None	,A = None	,A = None	,A = None	,A = None	,A = None	,A = None	,A = None	,A = None	,A = ChannelDimension.FIRST	,**A	,):
          UpperCAmelCase							=       do_resize if do_resize is not None else self.do_resize
          UpperCAmelCase							=       do_rescale if do_rescale is not None else self.do_rescale
          UpperCAmelCase							=       do_normalize if do_normalize is not None else self.do_normalize
          UpperCAmelCase							=       resample if resample is not None else self.resample
          UpperCAmelCase							=       rescale_factor if rescale_factor is not None else self.rescale_factor
          UpperCAmelCase							=       image_mean if image_mean is not None else self.image_mean
          UpperCAmelCase							=       image_std if image_std is not None else self.image_std

          UpperCAmelCase							=       size if size is not None else self.size
          UpperCAmelCase							=       get_size_dict(A					)

          UpperCAmelCase							=       make_list_of_images(A					)

          if not valid_images(A					):
                raise ValueError(
                    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                    """torch.Tensor, tf.Tensor or jax.ndarray."""					)

          if do_resize and size is None:
                raise ValueError("""Size must be specified if do_resize is True."""					)

          if do_rescale and rescale_factor is None:
                raise ValueError("""Rescale factor must be specified if do_rescale is True."""					)

          # All transformations expect numpy arrays.
          UpperCAmelCase							=       [to_numpy_array(A					) for image in images]

          if do_resize:
                UpperCAmelCase							=       [self.resize(image=A	,size=A	,resample=A					) for image in images]

          if do_rescale:
                UpperCAmelCase							=       [self.rescale(image=A	,scale=A					) for image in images]

          if do_normalize:
                UpperCAmelCase							=       [self.normalize(image=A	,mean=A	,std=A					) for image in images]

          UpperCAmelCase							=       [to_channel_dimension_format(A	,A					) for image in images]

          UpperCAmelCase							=       {"""pixel_values""": images}
          return BatchFeature(data=A	,tensor_type=A					)



