


"""simple docstring"""



import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case       =       16
__snake_case       =       32







def     __lowerCAmelCase		(		lowercase    :    int		,				lowercase    :    Union[str, Any] = 16					)      ->		List[str]:


    """simple docstring"""





    snake_case       : Tuple    =       AutoTokenizer.from_pretrained("bert-base-cased"					)
    snake_case       : Optional[int]    =       load_dataset("glue"		,				"mrpc"					)

    def tokenize_function(lowercase    :    List[str]					):
        # max_length=None => use the model max length (it's actually the default)
        snake_case       : Optional[Any]    =       tokenizer(examples["sentence1"]		,				examples["sentence2"]		,				truncation=lowercase_		,				max_length=lowercase_					)
        return outputs

    # Apply the method we just defined to all the examples in all the splits of the dataset
    # starting with the main process first:
    with accelerator.main_process_first():
        snake_case       : List[str]    =       datasets.map(
            lowercase_		,				batched=lowercase_		,				remove_columns=["idx", "sentence1", "sentence2"]		,				)

    # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
    # transformers library
    snake_case       : Union[str, Any]    =       tokenized_datasets.rename_column("label"		,				"labels"					)

    def collate_fn(lowercase    :    List[str]					):
        # On TPU it's best to pad everything to the same length or training will be very slow.
        snake_case       : Optional[int]    =       128 if accelerator.distributed_type == DistributedType.TPU else None
        # When using mixed precision we want round multiples of 8/16
        if accelerator.mixed_precision == "fp8":
            snake_case       : Dict    =       16
        elif accelerator.mixed_precision != "no":
            snake_case       : str    =       8
        else:
            snake_case       : Dict    =       None

        return tokenizer.pad(
            lowercase_		,				padding="longest"		,				max_length=lowercase_		,				pad_to_multiple_of=lowercase_		,				return_tensors="pt"		,				)

    # Instantiate dataloaders.
    snake_case       : int    =       DataLoader(
        tokenized_datasets["train"]		,				shuffle=lowercase_		,				collate_fn=lowercase_		,				batch_size=lowercase_					)
    snake_case       : Optional[int]    =       DataLoader(
        tokenized_datasets["validation"]		,				shuffle=lowercase_		,				collate_fn=lowercase_		,				batch_size=lowercase_					)

    return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  __snake_case       =       mocked_dataloaders  # noqa: F811







def     __lowerCAmelCase		(		lowercase    :    Any		,				lowercase    :    int					)      ->		Optional[Any]:


    """simple docstring"""





    if os.environ.get("TESTING_MOCKED_DATALOADERS"		,				lowercase_					) == "1":
        snake_case       : List[str]    =       2
    # New Code #
    snake_case       : List[str]    =       int(args.gradient_accumulation_steps					)
    # Initialize accelerator
    snake_case       : Optional[int]    =       Accelerator(
        cpu=args.cpu		,				mixed_precision=args.mixed_precision		,				gradient_accumulation_steps=lowercase_					)
    if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
        raise NotImplementedError(
            "Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`"					)
    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
    snake_case       : Dict    =       config["""lr"""]
    snake_case       : Dict    =       int(config["num_epochs"]					)
    snake_case       : int    =       int(config["seed"]					)
    snake_case       : int    =       int(config["batch_size"]					)

    snake_case       : str    =       evaluate.load("glue"		,				"mrpc"					)

    set_seed(lowercase_					)
    snake_case       : Dict    =       get_dataloaders(lowercase_		,				lowercase_					)
    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    snake_case       : int    =       AutoModelForSequenceClassification.from_pretrained("bert-base-cased"		,				return_dict=lowercase_					)

    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
    snake_case       : str    =       model.to(accelerator.device					)

    # Instantiate optimizer
    snake_case       : Any    =       AdamW(params=model.parameters()		,				lr=lowercase_					)

    # Instantiate scheduler
    snake_case       : Optional[int]    =       get_linear_schedule_with_warmup(
        optimizer=lowercase_		,				num_warmup_steps=100		,				num_training_steps=(len(lowercase_					) * num_epochs)		,				)

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    snake_case       : str    =       accelerator.prepare(
        lowercase_		,				lowercase_		,				lowercase_		,				lowercase_		,				lowercase_					)

    # Now we train the model
    for epoch in range(lowercase_					):
        model.train()
        for step, batch in enumerate(lowercase_					):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device					)
            # New code #
            # We use the new `accumulate` context manager to perform gradient accumulation
            # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
            with accelerator.accumulate(lowercase_					):
                snake_case       : List[str]    =       model(**lowercase_					)
                snake_case       : Any    =       output.loss
                accelerator.backward(lowercase_					)
                optimizer.step()
                lr_scheduler.step()
                optimizer.zero_grad()

        model.eval()
        for step, batch in enumerate(lowercase_					):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device					)
            with torch.no_grad():
                snake_case       : List[str]    =       model(**lowercase_					)
            snake_case       : Optional[Any]    =       outputs.logits.argmax(dim=-1					)
            snake_case       : Any    =       accelerator.gather_for_metrics((predictions, batch["labels"])					)
            metric.add_batch(
                predictions=lowercase_		,				references=lowercase_		,				)

        snake_case       : Optional[int]    =       metric.compute()
        # Use accelerator.print to print only on the main process.
        accelerator.print(F'epoch {epoch}:'		,				lowercase_					)







def     __lowerCAmelCase		(		)      ->		str:


    """simple docstring"""





    snake_case       : int    =       argparse.ArgumentParser(description="Simple example of training script."					)
    parser.add_argument(
        "--mixed_precision"		,				type=lowercase_		,				default=lowercase_		,				choices=["no", "fp16", "bf16", "fp8"]		,				help="Whether to use mixed precision. Choose"
        "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
        "and an Nvidia Ampere GPU."		,				)
    # New Code #
    parser.add_argument(
        "--gradient_accumulation_steps"		,				type=lowercase_		,				default=1		,				help="The number of minibatches to be ran before gradients are accumulated."		,				)
    parser.add_argument("--cpu"		,				action="store_true"		,				help="If passed, will train on the CPU."					)
    snake_case       : str    =       parser.parse_args()
    snake_case       : List[Any]    =       {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
    training_function(lowercase_		,				lowercase_					)


if __name__ == "__main__":
  main()







from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


lowerCamelCase__     :		Union[str, Any]        = logging.get_logger(__name__)



class    _snake_case (						UpperCAmelCase_				):
 __lowerCAmelCase	:  Any	   =      ['pixel_values']


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_ = True       ,	SCREAMING_SNAKE_CASE_ = 1 / 2_55       ,	SCREAMING_SNAKE_CASE_ = True       ,	SCREAMING_SNAKE_CASE_ = 8       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   super().__init__(**SCREAMING_SNAKE_CASE_)

   lowercase__	: List[str]     		=   do_rescale
   lowercase__	: List[Any]     		=   rescale_factor
   lowercase__	: Tuple     		=   do_pad
   lowercase__	: Optional[Any]     		=   pad_size


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return rescale(SCREAMING_SNAKE_CASE_       ,	scale=SCREAMING_SNAKE_CASE_       ,	data_format=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   lowercase__		,     lowercase__	: Optional[int]     		=   get_image_size(SCREAMING_SNAKE_CASE_)
   lowercase__	: Optional[Any]     		=   (old_height // size + 1) * size - old_height
   lowercase__	: str     		=   (old_width // size + 1) * size - old_width

   return pad(SCREAMING_SNAKE_CASE_       ,	((0, pad_height), (0, pad_width))       ,	mode="""symmetric"""       ,	data_format=SCREAMING_SNAKE_CASE_)



 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = ChannelDimension.FIRST       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   do_rescale if do_rescale is not None else self.do_rescale
   lowercase__	: int     		=   rescale_factor if rescale_factor is not None else self.rescale_factor
   lowercase__	: Union[str, Any]     		=   do_pad if do_pad is not None else self.do_pad
   lowercase__	: Optional[Any]     		=   pad_size if pad_size is not None else self.pad_size

   lowercase__	: str     		=   make_list_of_images(SCREAMING_SNAKE_CASE_)

   if not valid_images(SCREAMING_SNAKE_CASE_):
     raise ValueError(
         """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
         """torch.Tensor, tf.Tensor or jax.ndarray.""")

   if do_rescale and rescale_factor is None:
     raise ValueError("""Rescale factor must be specified if do_rescale is True.""")

   # All transformations expect numpy arrays.
   lowercase__	: List[Any]     		=   [to_numpy_array(SCREAMING_SNAKE_CASE_) for image in images]

   if do_rescale:
     lowercase__	: str     		=   [self.rescale(image=SCREAMING_SNAKE_CASE_       ,	scale=SCREAMING_SNAKE_CASE_) for image in images]

   if do_pad:
     lowercase__	: List[str]     		=   [self.pad(SCREAMING_SNAKE_CASE_       ,	size=SCREAMING_SNAKE_CASE_) for image in images]

   lowercase__	: Optional[Any]     		=   [to_channel_dimension_format(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_) for image in images]

   lowercase__	: Dict     		=   {"""pixel_values""": images}
   return BatchFeature(data=SCREAMING_SNAKE_CASE_       ,	tensor_type=SCREAMING_SNAKE_CASE_)






