


import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


snake_case__				:	Any					       =					16
snake_case__				:	List[Any]					       =					32


def    __lowerCamelCase   (      A__						: Accelerator						,						A__						: int = 16					)     ->    Optional[Any]:
   lowerCamelCase_						:   Optional[int]		= AutoTokenizer.from_pretrained("""bert-base-cased"""					)
   lowerCamelCase_						:   Optional[Any]		= load_dataset("""glue"""						,						"""mrpc"""					)

   def tokenize_function(A__						: str					):
      # max_length=None => use the model max length (it's actually the default)
      lowerCamelCase_						:   Any		= tokenizer(examples["""sentence1"""]						,						examples["""sentence2"""]						,						truncation=A__						,						max_length=A__					)
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      lowerCamelCase_						:   Optional[Any]		= datasets.map(
          A__						,						batched=A__						,						remove_columns=["""idx""", """sentence1""", """sentence2"""]						,						)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   lowerCamelCase_						:   Optional[int]		= tokenized_datasets.rename_column("""label"""						,						"""labels"""					)

   def collate_fn(A__						: Optional[int]					):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      lowerCamelCase_						:   int		= 128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         lowerCamelCase_						:   Optional[int]		= 16
      elif accelerator.mixed_precision != "no":
         lowerCamelCase_						:   Optional[int]		= 8
      else:
         lowerCamelCase_						:   List[Any]		= None

      return tokenizer.pad(
          A__						,						padding="""longest"""						,						max_length=A__						,						pad_to_multiple_of=A__						,						return_tensors="""pt"""						,						)

   # Instantiate dataloaders.
   lowerCamelCase_						:   List[str]		= DataLoader(
       tokenized_datasets["""train"""]						,						shuffle=A__						,						collate_fn=A__						,						batch_size=A__					)
   lowerCamelCase_						:   List[Any]		= DataLoader(
       tokenized_datasets["""validation"""]						,						shuffle=A__						,						collate_fn=A__						,						batch_size=A__					)

   return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      snake_case__				:	Any					       =					mocked_dataloaders  # noqa: F811


def    __lowerCamelCase   (      A__						: Optional[int]						,						A__						: Tuple					)     ->    Tuple:
   # For testing only
   if os.environ.get("""TESTING_MOCKED_DATALOADERS"""						,						A__					) == "1":
      lowerCamelCase_						:   str		= 2
   # New Code #
   lowerCamelCase_						:   List[str]		= int(args.gradient_accumulation_steps					)
   lowerCamelCase_						:   Dict		= int(args.local_sgd_steps					)
   # Initialize accelerator
   lowerCamelCase_						:   Dict		= Accelerator(
       cpu=args.cpu						,						mixed_precision=args.mixed_precision						,						gradient_accumulation_steps=A__					)
   if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
      raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"""					)
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   lowerCamelCase_						:   Tuple		= config["""lr"""]
   lowerCamelCase_						:   str		= int(config["""num_epochs"""]					)
   lowerCamelCase_						:   Union[str, Any]		= int(config["""seed"""]					)
   lowerCamelCase_						:   int		= int(config["""batch_size"""]					)

   lowerCamelCase_						:   List[Any]		= evaluate.load("""glue"""						,						"""mrpc"""					)

   set_seed(A__					)
   lowerCamelCase_,	lowerCamelCase_						:   Tuple		= get_dataloaders(A__						,						A__					)
   # Instantiate the model (we build the model here so that the seed also control new weights initialization)
   lowerCamelCase_						:   Optional[int]		= AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""						,						return_dict=A__					)

   # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
   # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
   # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
   lowerCamelCase_						:   Any		= model.to(accelerator.device					)

   # Instantiate optimizer
   lowerCamelCase_						:   List[Any]		= AdamW(params=model.parameters()						,						lr=A__					)

   # Instantiate scheduler
   lowerCamelCase_						:   List[Any]		= get_linear_schedule_with_warmup(
       optimizer=A__						,						num_warmup_steps=100						,						num_training_steps=(len(A__					) * num_epochs)						,						)

   # Prepare everything
   # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
   # prepare method.
   lowerCamelCase_,	lowerCamelCase_,	lowerCamelCase_,	lowerCamelCase_,	lowerCamelCase_						:   str		= accelerator.prepare(
       A__						,						A__						,						A__						,						A__						,						A__					)

   # Now we train the model
   for epoch in range(A__					):
      model.train()
      with LocalSGD(
          accelerator=A__						,						model=A__						,						local_sgd_steps=A__						,						enabled=local_sgd_steps is not None					) as local_sgd:
         for step, batch in enumerate(A__					):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device					)
            # New code #
            # We use the new `accumulate` context manager to perform gradient accumulation
            # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
            with accelerator.accumulate(A__					):
               lowerCamelCase_						:   str		= model(**A__					)
               lowerCamelCase_						:   str		= output.loss
               accelerator.backward(A__					)
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()
               # LocalSGD-specific line
               local_sgd.step()

      model.eval()
      for step, batch in enumerate(A__					):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device					)
         with torch.no_grad():
            lowerCamelCase_						:   str		= model(**A__					)
         lowerCamelCase_						:   List[str]		= outputs.logits.argmax(dim=-1					)
         lowerCamelCase_,	lowerCamelCase_						:   List[Any]		= accelerator.gather_for_metrics((predictions, batch["""labels"""])					)
         metric.add_batch(
             predictions=A__						,						references=A__						,						)

      lowerCamelCase_						:   Optional[Any]		= metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(f'''epoch {epoch}:'''						,						A__					)


def    __lowerCamelCase   (      )     ->    Tuple:
   lowerCamelCase_						:   int		= argparse.ArgumentParser(description="""Simple example of training script."""					)
   parser.add_argument(
       """--mixed_precision"""						,						type=A__						,						default=A__						,						choices=["""no""", """fp16""", """bf16""", """fp8"""]						,						help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""						,						)
   # New Code #
   parser.add_argument(
       """--gradient_accumulation_steps"""						,						type=A__						,						default=1						,						help="""The number of minibatches to be ran before gradients are accumulated."""						,						)
   parser.add_argument(
       """--local_sgd_steps"""						,						type=A__						,						default=8						,						help="""Number of local SGD steps or None to disable local SGD"""					)
   parser.add_argument("""--cpu"""						,						action="""store_true"""						,						help="""If passed, will train on the CPU."""					)
   lowerCamelCase_						:   int		= parser.parse_args()
   lowerCamelCase_						:   Optional[Any]		= {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
   training_function(A__						,						A__					)


if __name__ == "__main__":
      main()









# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
      import torch


snake_case__				:	int					       =					logging.get_logger(__name__)





@dataclass
class   SCREAMING_SNAKE_CASE_			:




       '''simple docstring'''



       def __init__(						self				:				int		,				__a				:				Optional[Any]=False		,				__a				:				Tuple=False		,				__a				:				Tuple=6.0		,				__a				:				str=None		,				__a				:				List[str]=False		,				__a				:				str=False		,				__a				:				Optional[Any]=None		,				__a				:				Dict="fp4"		,				__a				:				Optional[int]=False		,				**__a				:				Dict		,				)					->str:
          lowerCamelCase_						:   str		= load_in_abit
          lowerCamelCase_						:   Union[str, Any]		= load_in_abit
          lowerCamelCase_						:   Tuple		= llm_inta_threshold
          lowerCamelCase_						:   int		= llm_inta_skip_modules
          lowerCamelCase_						:   List[str]		= llm_inta_enable_fpaa_cpu_offload
          lowerCamelCase_						:   Optional[Any]		= llm_inta_has_fpaa_weight
          lowerCamelCase_						:   Dict		= bnb_abit_quant_type
          lowerCamelCase_						:   int		= bnb_abit_use_double_quant

          if bnb_abit_compute_dtype is None:
             lowerCamelCase_						:   int		= torch.floataa
          elif isinstance(__a		,				__a							):
             lowerCamelCase_						:   Tuple		= getattr(__a		,				__a							)
          elif isinstance(__a		,				torch.dtype							):
             lowerCamelCase_						:   List[str]		= bnb_abit_compute_dtype
          else:
             raise ValueError("""bnb_4bit_compute_dtype must be a string or a torch.dtype"""							)

          self.post_init()



       def 				_lowerCAmelCase							(						self				:				Any							)					->Union[str, Any]:
          if not isinstance(self.llm_inta_threshold		,				__a							):
             raise ValueError("""llm_int8_threshold must be a float"""							)

          if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules		,				__a							):
             raise ValueError("""llm_int8_skip_modules must be a list of strings"""							)
          if not isinstance(self.llm_inta_enable_fpaa_cpu_offload		,				__a							):
             raise ValueError("""llm_int8_enable_fp32_cpu_offload must be a boolean"""							)

          if not isinstance(self.llm_inta_has_fpaa_weight		,				__a							):
             raise ValueError("""llm_int8_has_fp16_weight must be a boolean"""							)

          if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype		,				torch.dtype							):
             raise ValueError("""bnb_4bit_compute_dtype must be torch.dtype"""							)

          if not isinstance(self.bnb_abit_quant_type		,				__a							):
             raise ValueError("""bnb_4bit_quant_type must be a string"""							)

          if not isinstance(self.bnb_abit_use_double_quant		,				__a							):
             raise ValueError("""bnb_4bit_use_double_quant must be a boolean"""							)

          if self.load_in_abit and not version.parse(importlib.metadata.version("""bitsandbytes"""							)							) >= version.parse(
              """0.39.0"""							):
             raise ValueError(
                 """4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"""							)



       def 				_lowerCAmelCase							(						self				:				Optional[int]							)					->int:
          return self.load_in_abit or self.load_in_abit



       def 				_lowerCAmelCase							(						self				:				List[str]							)					->List[Any]:
          if self.load_in_abit:
             return "llm_int8"
          elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
             return "fp4"
          elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
             return "nf4"
          else:
             return None



       @classmethod
       def 				_lowerCAmelCase							(						cls				:				int		,				__a				:				Dict		,				__a				:				str		,				**__a				:				Any							)					->Optional[Any]:
          lowerCamelCase_						:   Optional[Any]		= cls(**__a							)

          lowerCamelCase_						:   List[Any]		= []
          for key, value in kwargs.items():
             if hasattr(__a		,				__a							):
                setattr(__a		,				__a		,				__a							)
                to_remove.append(__a							)
          for key in to_remove:
             kwargs.pop(__a		,				__a							)

          if return_unused_kwargs:
             return config, kwargs
          else:
             return config



       def 				_lowerCAmelCase							(						self				:				Optional[Any]		,				__a				:				Union[str, os.PathLike]							)					->Any:
          with open(__a		,				"""w"""		,				encoding="""utf-8"""							) as writer:
             lowerCamelCase_						:   List[Any]		= self.to_dict()
             lowerCamelCase_						:   Dict		= json.dumps(__a		,				indent=2		,				sort_keys=__a							) + """\n"""

             writer.write(__a							)



       def 				_lowerCAmelCase							(						self				:				Tuple							)					->Dict[str, Any]:
          lowerCamelCase_						:   Dict		= copy.deepcopy(self.__dict__							)
          lowerCamelCase_						:   Tuple		= str(output["""bnb_4bit_compute_dtype"""]							).split("""."""							)[1]

          return output



       def __repr__(						self				:				List[str]							)					->Any:
          return F'''{self.__class__.__name__} {self.to_json_string()}'''



       def 				_lowerCAmelCase							(						self				:				Optional[int]		,				__a				:				bool = True							)					->str:
          if use_diff is True:
             lowerCamelCase_						:   Dict		= self.to_diff_dict()
          else:
             lowerCamelCase_						:   Any		= self.to_dict()
          return json.dumps(__a		,				indent=2		,				sort_keys=__a							) + "\n"




       def 				_lowerCAmelCase							(						self				:				Optional[Any]							)					->Dict[str, Any]:
          lowerCamelCase_						:   Any		= self.to_dict()

          # get the default config dict
          lowerCamelCase_						:   List[str]		= BitsAndBytesConfig().to_dict()

          lowerCamelCase_						:   str		= {}

          # only serialize values that differ from the default config
          for key, value in config_dict.items():
             if value != default_config_dict[key]:
                lowerCamelCase_						:   List[Any]		= value

          return serializable_config_dict





