






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCAmelCase__							  =						16
UpperCAmelCase__							  =						32
def 		_a       ( a					:Accelerator      ,				a					:int = 16					)				->     Union[str, Any]:
 a						    = AutoTokenizer.from_pretrained('''bert-base-cased'''					)
 a						    = load_dataset('''glue'''      ,				'''mrpc'''					)

 def tokenize_function(a					:Optional[int]					):
  # max_length=None => use the model max length (it's actually the default)
  a						    = tokenizer(examples['''sentence1''']      ,				examples['''sentence2''']      ,				truncation=a      ,				max_length=a					)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  a						    = datasets.map(
      a      ,				batched=a      ,				remove_columns=['''idx''', '''sentence1''', '''sentence2''']      ,				)

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 a						    = tokenized_datasets.rename_column('''label'''      ,				'''labels'''					)

 def collate_fn(a					:List[Any]					):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  a						    = 128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   a						    = 16
  elif accelerator.mixed_precision != "no":
   a						    = 8
  else:
   a						    = None

  return tokenizer.pad(
      a      ,				padding='''longest'''      ,				max_length=a      ,				pad_to_multiple_of=a      ,				return_tensors='''pt'''      ,				)

 # Instantiate dataloaders.
 a						    = DataLoader(
     tokenized_datasets['''train''']      ,				shuffle=a      ,				collate_fn=a      ,				batch_size=a					)
 a						    = DataLoader(
     tokenized_datasets['''validation''']      ,				shuffle=a      ,				collate_fn=a      ,				batch_size=a					)

 return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
     from accelerate.test_utils.training import mocked_dataloaders

     UpperCAmelCase__							  =						mocked_dataloaders  # noqa: F811
def 		_a       ( a					:List[str]      ,				a					:str					)				->     Optional[Any]:
 # For testing only
 if os.environ.get('''TESTING_MOCKED_DATALOADERS'''      ,				a					) == "1":
  a						    = 2
 # New Code #
 a						    = int(args.gradient_accumulation_steps					)
 # Initialize accelerator
 a						    = Accelerator(
     cpu=args.cpu      ,				mixed_precision=args.mixed_precision      ,				gradient_accumulation_steps=a					)
 if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
  raise NotImplementedError(
      '''Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`'''					)
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 a						    = config['''lr''']
 a						    = int(config['''num_epochs''']					)
 a						    = int(config['''seed''']					)
 a						    = int(config['''batch_size''']					)

 a						    = evaluate.load('''glue'''      ,				'''mrpc'''					)

 set_seed(a					)
 a    ,	a						    = get_dataloaders(a      ,				a					)
 # Instantiate the model (we build the model here so that the seed also control new weights initialization)
 a						    = AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''      ,				return_dict=a					)

 # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
 # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
 # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
 a						    = model.to(accelerator.device					)

 # Instantiate optimizer
 a						    = AdamW(params=model.parameters()      ,				lr=a					)

 # Instantiate scheduler
 a						    = get_linear_schedule_with_warmup(
     optimizer=a      ,				num_warmup_steps=100      ,				num_training_steps=(len(a					) * num_epochs)      ,				)

 # Prepare everything
 # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
 # prepare method.
 a    ,	a    ,	a    ,	a    ,	a						    = accelerator.prepare(
     a      ,				a      ,				a      ,				a      ,				a					)

 # Now we train the model
 for epoch in range(a					):
  model.train()
  for step, batch in enumerate(a					):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device					)
   # New code #
   # We use the new `accumulate` context manager to perform gradient accumulation
   # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
   with accelerator.accumulate(a					):
    a						    = model(**a					)
    a						    = output.loss
    accelerator.backward(a					)
    optimizer.step()
    lr_scheduler.step()
    optimizer.zero_grad()

  model.eval()
  for step, batch in enumerate(a					):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device					)
   with torch.no_grad():
    a						    = model(**a					)
   a						    = outputs.logits.argmax(dim=-1					)
   a    ,	a						    = accelerator.gather_for_metrics((predictions, batch['''labels'''])					)
   metric.add_batch(
       predictions=a      ,				references=a      ,				)

  a						    = metric.compute()
  # Use accelerator.print to print only on the main process.
  accelerator.print(F"""epoch {epoch}:"""      ,				a					)
def 		_a       ( )				->     int:
 a						    = argparse.ArgumentParser(description='''Simple example of training script.'''					)
 parser.add_argument(
     '''--mixed_precision'''      ,				type=a      ,				default=a      ,				choices=['''no''', '''fp16''', '''bf16''', '''fp8''']      ,				help='''Whether to use mixed precision. Choose'''
     '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
     '''and an Nvidia Ampere GPU.'''      ,				)
 # New Code #
 parser.add_argument(
     '''--gradient_accumulation_steps'''      ,				type=a      ,				default=1      ,				help='''The number of minibatches to be ran before gradients are accumulated.'''      ,				)
 parser.add_argument('''--cpu'''      ,				action='''store_true'''      ,				help='''If passed, will train on the CPU.'''					)
 a						    = parser.parse_args()
 a						    = {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
 training_function(a      ,				a					)


if __name__ == "__main__":
     main()










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase__							  =						{"processing_layoutxlm": ["LayoutXLMProcessor"]}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCAmelCase__							  =						["LayoutXLMTokenizer"]

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCAmelCase__							  =						["LayoutXLMTokenizerFast"]

if TYPE_CHECKING:
     from .processing_layoutxlm import LayoutXLMProcessor

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_layoutxlm import LayoutXLMTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_layoutxlm_fast import LayoutXLMTokenizerFast

else:
     import sys

     UpperCAmelCase__							  =						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


