

import math
import os
from copy import deepcopy

import datasets
import evaluate
import torch
import transformers
from datasets import load_dataset
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer

from accelerate import Accelerator
from accelerate.test_utils import RegressionDataset, RegressionModel
from accelerate.utils import is_tpu_available, set_seed


lowercase_			=  "true"
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Optional[Any]   ,      SCREAMING_SNAKE_CASE__      :     Dict=82   ,      SCREAMING_SNAKE_CASE__      :     Union[str, Any]=16				)							->				List[str]:





    '''simple docstring'''






    set_seed(42				)
    A__   				=       RegressionModel()
    A__   				=       deepcopy(SCREAMING_SNAKE_CASE__				)
    A__   				=       RegressionDataset(length=SCREAMING_SNAKE_CASE__				)
    A__   				=       DataLoader(SCREAMING_SNAKE_CASE__   ,      batch_size=SCREAMING_SNAKE_CASE__				)
    model.to(accelerator.device				)
    A__				,						A__   				=       accelerator.prepare(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
    return model, ddp_model, dataloader
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Accelerator   ,      SCREAMING_SNAKE_CASE__      :     str=False				)							->				int:





    '''simple docstring'''






    A__   				=       AutoTokenizer.from_pretrained('hf-internal-testing/mrpc-bert-base-cased'				)
    A__   				=       load_dataset('glue'   ,      'mrpc'   ,      split='validation'				)

    def tokenize_function(SCREAMING_SNAKE_CASE__      :     Union[str, Any]				):
        A__   				=       tokenizer(examples['sentence1']   ,      examples['sentence2']   ,      truncation=SCREAMING_SNAKE_CASE__   ,      max_length=SCREAMING_SNAKE_CASE__				)
        return outputs

    with accelerator.main_process_first():
        A__   				=       dataset.map(
            SCREAMING_SNAKE_CASE__   ,      batched=SCREAMING_SNAKE_CASE__   ,      remove_columns=['idx', 'sentence1', 'sentence2']   ,      )

    A__   				=       tokenized_datasets.rename_column('label'   ,      'labels'				)

    def collate_fn(SCREAMING_SNAKE_CASE__      :     List[str]				):
        if use_longest:
            return tokenizer.pad(SCREAMING_SNAKE_CASE__   ,      padding='longest'   ,      return_tensors='pt'				)
        return tokenizer.pad(SCREAMING_SNAKE_CASE__   ,      padding='max_length'   ,      max_length=128   ,      return_tensors='pt'				)

    return DataLoader(SCREAMING_SNAKE_CASE__   ,      shuffle=SCREAMING_SNAKE_CASE__   ,      collate_fn=SCREAMING_SNAKE_CASE__   ,      batch_size=16				)
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     int   ,      SCREAMING_SNAKE_CASE__      :     List[Any]				)							->				Tuple:





    '''simple docstring'''






    A__   				=       Accelerator(dispatch_batches=SCREAMING_SNAKE_CASE__   ,      split_batches=SCREAMING_SNAKE_CASE__				)
    A__   				=       get_dataloader(SCREAMING_SNAKE_CASE__   ,      not dispatch_batches				)
    A__   				=       AutoModelForSequenceClassification.from_pretrained(
        'hf-internal-testing/mrpc-bert-base-cased'   ,      return_dict=SCREAMING_SNAKE_CASE__				)
    A__				,						A__   				=       accelerator.prepare(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
    return {"ddp": [ddp_model, ddp_dataloader, "cuda:0"], "no": [model, dataloader, accelerator.device]}, accelerator
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Optional[Any]   ,      SCREAMING_SNAKE_CASE__      :     Tuple   ,      SCREAMING_SNAKE_CASE__      :     int				)							->				List[Any]:





    '''simple docstring'''






    A__   				=       []
    for batch in dataloader:
        A__				,						A__   				=       batch.values()
        with torch.no_grad():
            A__   				=       model(SCREAMING_SNAKE_CASE__				)
            A__				,						A__   				=       accelerator.gather_for_metrics((logit, target)				)
            logits_and_targets.append((logit, target)				)
    A__				,						A__   				=       [], []
    for logit, targ in logits_and_targets:
        logits.append(SCREAMING_SNAKE_CASE__				)
        targs.append(SCREAMING_SNAKE_CASE__				)
    A__				,						A__   				=       torch.cat(SCREAMING_SNAKE_CASE__				), torch.cat(SCREAMING_SNAKE_CASE__				)
    return logits, targs
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Accelerator   ,      SCREAMING_SNAKE_CASE__      :     List[Any]=82   ,      SCREAMING_SNAKE_CASE__      :     Any=False   ,      SCREAMING_SNAKE_CASE__      :     str=False   ,      SCREAMING_SNAKE_CASE__      :     Dict=16				)							->				Any:





    '''simple docstring'''






    A__				,						A__				,						A__   				=       get_basic_setup(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
    A__				,						A__   				=       generate_predictions(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
    assert (
        len(SCREAMING_SNAKE_CASE__				) == num_samples
    ), f'Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(SCREAMING_SNAKE_CASE__				)}'
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     bool = False   ,      SCREAMING_SNAKE_CASE__      :     bool = False				)							->				Dict:





    '''simple docstring'''






    A__   				=       evaluate.load('glue'   ,      'mrpc'				)
    A__				,						A__   				=       get_mrpc_setup(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
    # First do baseline
    A__				,						A__				,						A__   				=       setup['no']
    model.to(SCREAMING_SNAKE_CASE__				)
    model.eval()
    for batch in dataloader:
        batch.to(SCREAMING_SNAKE_CASE__				)
        with torch.inference_mode():
            A__   				=       model(**SCREAMING_SNAKE_CASE__				)
        A__   				=       outputs.logits.argmax(dim=-1				)
        metric.add_batch(predictions=SCREAMING_SNAKE_CASE__   ,      references=batch['labels']				)
    A__   				=       metric.compute()

    # Then do distributed
    A__				,						A__				,						A__   				=       setup['ddp']
    model.eval()
    for batch in dataloader:
        with torch.inference_mode():
            A__   				=       model(**SCREAMING_SNAKE_CASE__				)
        A__   				=       outputs.logits.argmax(dim=-1				)
        A__   				=       batch['labels']
        A__				,						A__   				=       accelerator.gather_for_metrics((preds, references)				)
        metric.add_batch(predictions=SCREAMING_SNAKE_CASE__   ,      references=SCREAMING_SNAKE_CASE__				)
    A__   				=       metric.compute()

    for key in "accuracy f1".split():
        assert math.isclose(
            baseline[key]   ,      distributed[key]				), f'Baseline and Distributed are not the same for key {key}:\n\tBaseline: {baseline[key]}\n\tDistributed: {distributed[key]}\n'
def 						_snake_case(   )							->				Union[str, Any]:





    '''simple docstring'''






    A__   				=       Accelerator(split_batches=SCREAMING_SNAKE_CASE__   ,      dispatch_batches=SCREAMING_SNAKE_CASE__				)
    if accelerator.is_local_main_process:
        datasets.utils.logging.set_verbosity_warning()
        transformers.utils.logging.set_verbosity_warning()
    else:
        datasets.utils.logging.set_verbosity_error()
        transformers.utils.logging.set_verbosity_error()
    # These are a bit slower so they should only be ran on the GPU or TPU
    if torch.cuda.is_available() or is_tpu_available():
        if accelerator.is_local_main_process:
            print('**Testing gather_for_metrics**'				)
        for split_batches in [True, False]:
            for dispatch_batches in [True, False]:
                if accelerator.is_local_main_process:
                    print(f'With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`'				)
                test_mrpc(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)
                accelerator.state._reset_state()
    if accelerator.is_local_main_process:
        print('**Test torch metrics**'				)
    for split_batches in [True, False]:
        for dispatch_batches in [True, False]:
            A__   				=       Accelerator(split_batches=SCREAMING_SNAKE_CASE__   ,      dispatch_batches=SCREAMING_SNAKE_CASE__				)
            if accelerator.is_local_main_process:
                print(f'With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`, length=99'				)
            test_torch_metrics(SCREAMING_SNAKE_CASE__   ,      99				)
            accelerator.state._reset_state()
    if accelerator.is_local_main_process:
        print('**Test last batch is not dropped when perfectly divisible**'				)
    A__   				=       Accelerator()
    test_torch_metrics(SCREAMING_SNAKE_CASE__   ,      512				)
    accelerator.state._reset_state()
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     int				)							->				List[str]:





    '''simple docstring'''






    main()


if __name__ == "__main__":
     main()




import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)


class 			A						(							nn.Module			):







    """simple docstring"""





    def __init__(				self			:	Union[str, Any]		)->   List[str]:





        '''simple docstring'''







        super().__init__()
        A__   				=       nn.Linear(3,4		)
        A__   				=       nn.BatchNormad(4		)
        A__   				=       nn.Linear(4,5		)


    def 						snake_case__					(				self			:	Dict,lowercase_			:	Union[str, Any]		)->   Tuple:





        '''simple docstring'''







        return self.lineara(self.batchnorm(self.lineara(lowercase_		)		)		)




class 			A						(							unittest.TestCase			):







    """simple docstring"""





    def 						snake_case__					(				self			:	List[str]		)->   Any:





        '''simple docstring'''







        A__   				=       ModelForTest()
        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,model.state_dict()		)
            A__   				=       os.path.join(lowercase_,'index.json'		)
            self.assertTrue(os.path.isfile(lowercase_		)		)
            # TODO: add tests on what is inside the index

            for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
                A__   				=       os.path.join(lowercase_,F'{key}.dat'		)
                self.assertTrue(os.path.isfile(lowercase_		)		)
                # TODO: add tests on the fact weights are properly loaded


    def 						snake_case__					(				self			:	List[Any]		)->   Optional[int]:





        '''simple docstring'''







        A__   				=       [torch.floataa, torch.floataa, torch.bfloataa]

        for dtype in dtypes:
            A__   				=       torch.randn(2,3,dtype=lowercase_		)
            with TemporaryDirectory() as tmp_dir:
                A__   				=       offload_weight(lowercase_,'weight',lowercase_,{}		)
                A__   				=       os.path.join(lowercase_,'weight.dat'		)
                self.assertTrue(os.path.isfile(lowercase_		)		)
                self.assertDictEqual(lowercase_,{'weight': {'shape': [2, 3], 'dtype': str(lowercase_		).split('.'		)[1]}}		)

                A__   				=       load_offloaded_weight(lowercase_,index['weight']		)
                self.assertTrue(torch.equal(lowercase_,lowercase_		)		)


    def 						snake_case__					(				self			:	Optional[int]		)->   Optional[Any]:





        '''simple docstring'''







        A__   				=       ModelForTest()
        A__   				=       model.state_dict()
        A__   				=       {k: v for k, v in state_dict.items() if 'linear2' not in k}
        A__   				=       {k: v for k, v in state_dict.items() if 'linear2' in k}

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)

        A__   				=       {k: v for k, v in state_dict.items() if 'weight' in k}
        A__   				=       {k: v for k, v in state_dict.items() if 'weight' not in k}

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            # Duplicates are removed
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)


    def 						snake_case__					(				self			:	int		)->   Union[str, Any]:





        '''simple docstring'''







        A__   				=       {'a.1': 0, 'a.10': 1, 'a.2': 2}
        A__   				=       extract_submodules_state_dict(lowercase_,['a.1', 'a.2']		)
        self.assertDictEqual(lowercase_,{'a.1': 0, 'a.2': 2}		)

        A__   				=       {'a.1.a': 0, 'a.10.a': 1, 'a.2.a': 2}
        A__   				=       extract_submodules_state_dict(lowercase_,['a.1', 'a.2']		)
        self.assertDictEqual(lowercase_,{'a.1.a': 0, 'a.2.a': 2}		)

