'''simple docstring'''

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class     __UpperCAmelCase       :





  def __init__(	self    ,     _lowerCamelCase    ,     _lowerCamelCase=13    ,     _lowerCamelCase=7    ,     _lowerCamelCase=True    ,     _lowerCamelCase=True    ,     _lowerCamelCase=True    ,     _lowerCamelCase=True    ,     _lowerCamelCase=99    ,     _lowerCamelCase=16    ,     _lowerCamelCase=36    ,     _lowerCamelCase=6    ,     _lowerCamelCase=6    ,     _lowerCamelCase=6    ,     _lowerCamelCase=37    ,     _lowerCamelCase="gelu"    ,     _lowerCamelCase=0.1    ,     _lowerCamelCase=0.1    ,     _lowerCamelCase=512    ,     _lowerCamelCase=16    ,     _lowerCamelCase=2    ,     _lowerCamelCase=0.02    ,     _lowerCamelCase=3    ,     _lowerCamelCase=4    ,     _lowerCamelCase=None    ,     ):
     lowerCAmelCase_			=      parent
     lowerCAmelCase_			=      batch_size
     lowerCAmelCase_			=      seq_length
     lowerCAmelCase_			=      is_training
     lowerCAmelCase_			=      use_input_mask
     lowerCAmelCase_			=      use_token_type_ids
     lowerCAmelCase_			=      use_labels
     lowerCAmelCase_			=      vocab_size
     lowerCAmelCase_			=      embedding_size
     lowerCAmelCase_			=      hidden_size
     lowerCAmelCase_			=      num_hidden_layers
     lowerCAmelCase_			=      num_hidden_groups
     lowerCAmelCase_			=      num_attention_heads
     lowerCAmelCase_			=      intermediate_size
     lowerCAmelCase_			=      hidden_act
     lowerCAmelCase_			=      hidden_dropout_prob
     lowerCAmelCase_			=      attention_probs_dropout_prob
     lowerCAmelCase_			=      max_position_embeddings
     lowerCAmelCase_			=      type_vocab_size
     lowerCAmelCase_			=      type_sequence_label_size
     lowerCAmelCase_			=      initializer_range
     lowerCAmelCase_			=      num_labels
     lowerCAmelCase_			=      num_choices
     lowerCAmelCase_			=      scope





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      ids_tensor([self.batch_size, self.seq_length]    ,     self.vocab_size )

     lowerCAmelCase_			=      None
     if self.use_input_mask:
        lowerCAmelCase_			=      random_attention_mask([self.batch_size, self.seq_length] )

     lowerCAmelCase_			=      None
     if self.use_token_type_ids:
        lowerCAmelCase_			=      ids_tensor([self.batch_size, self.seq_length]    ,     self.type_vocab_size )

     lowerCAmelCase_			=      None
     lowerCAmelCase_			=      None
     lowerCAmelCase_			=      None
     if self.use_labels:
        lowerCAmelCase_			=      ids_tensor([self.batch_size]    ,     self.type_sequence_label_size )
        lowerCAmelCase_			=      ids_tensor([self.batch_size, self.seq_length]    ,     self.num_labels )
        lowerCAmelCase_			=      ids_tensor([self.batch_size]    ,     self.num_choices )

     lowerCAmelCase_			=      self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





  def 						UpperCAmelCase_	(	self ):
     return AlbertConfig(
         vocab_size=self.vocab_size    ,     hidden_size=self.hidden_size    ,     num_hidden_layers=self.num_hidden_layers    ,     num_attention_heads=self.num_attention_heads    ,     intermediate_size=self.intermediate_size    ,     hidden_act=self.hidden_act    ,     hidden_dropout_prob=self.hidden_dropout_prob    ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,     max_position_embeddings=self.max_position_embeddings    ,     type_vocab_size=self.type_vocab_size    ,     initializer_range=self.initializer_range    ,     num_hidden_groups=self.num_hidden_groups    ,     )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      AlbertModel(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(_lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase )
     lowerCAmelCase_			=      model(_lowerCamelCase    ,     token_type_ids=_lowerCamelCase )
     lowerCAmelCase_			=      model(_lowerCamelCase )
     self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.hidden_size) )
     self.parent.assertEqual(result.pooler_output.shape    ,     (self.batch_size, self.hidden_size) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      AlbertForPreTraining(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(
         _lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     labels=_lowerCamelCase    ,     sentence_order_label=_lowerCamelCase    ,     )
     self.parent.assertEqual(result.prediction_logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size) )
     self.parent.assertEqual(result.sop_logits.shape    ,     (self.batch_size, config.num_labels) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      AlbertForMaskedLM(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(_lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     labels=_lowerCamelCase )
     self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      AlbertForQuestionAnswering(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(
         _lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     start_positions=_lowerCamelCase    ,     end_positions=_lowerCamelCase    ,     )
     self.parent.assertEqual(result.start_logits.shape    ,     (self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape    ,     (self.batch_size, self.seq_length) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      self.num_labels
     lowerCAmelCase_			=      AlbertForSequenceClassification(_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(_lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     labels=_lowerCamelCase )
     self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_labels) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      self.num_labels
     lowerCAmelCase_			=      AlbertForTokenClassification(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      model(_lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     labels=_lowerCamelCase )
     self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.num_labels) )





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
     lowerCAmelCase_			=      self.num_choices
     lowerCAmelCase_			=      AlbertForMultipleChoice(config=_lowerCamelCase )
     model.to(_lowerCamelCase )
     model.eval()
     lowerCAmelCase_			=      input_ids.unsqueeze(1 ).expand(-1    ,     self.num_choices    ,     -1 ).contiguous()
     lowerCAmelCase_			=      token_type_ids.unsqueeze(1 ).expand(-1    ,     self.num_choices    ,     -1 ).contiguous()
     lowerCAmelCase_			=      input_mask.unsqueeze(1 ).expand(-1    ,     self.num_choices    ,     -1 ).contiguous()
     lowerCAmelCase_			=      model(
         _lowerCamelCase    ,     attention_mask=_lowerCamelCase    ,     token_type_ids=_lowerCamelCase    ,     labels=_lowerCamelCase    ,     )
     self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_choices) )







  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.prepare_config_and_inputs()
     (
         (
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,
     )			=      config_and_inputs
     lowerCAmelCase_			=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict






@require_torch
class     __UpperCAmelCase       ( __a						,       __a						,       unittest.TestCase		):
  __A :				Tuple											= (
      (
          AlbertModel,
          AlbertForPreTraining,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  __A :				Tuple											= (
      {
          'feature-extraction': AlbertModel,
          'fill-mask': AlbertForMaskedLM,
          'question-answering': AlbertForQuestionAnswering,
          'text-classification': AlbertForSequenceClassification,
          'token-classification': AlbertForTokenClassification,
          'zero-shot': AlbertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  __A :				List[str]											= True





  def 						UpperCAmelCase_	(	self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase=False ):
     lowerCAmelCase_			=      super()._prepare_for_class(_lowerCamelCase    ,     _lowerCamelCase    ,     return_labels=_lowerCamelCase )

     if return_labels:
        if model_class in get_values(_lowerCamelCase ):
           lowerCAmelCase_			=      torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)    ,     dtype=torch.long    ,     device=_lowerCamelCase )
           lowerCAmelCase_			=      torch.zeros(
               self.model_tester.batch_size    ,     dtype=torch.long    ,     device=_lowerCamelCase )
     return inputs_dict





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      AlbertModelTester(self )
     lowerCAmelCase_			=      ConfigTester(self    ,     config_class=_lowerCamelCase    ,     hidden_size=37 )





  def 						UpperCAmelCase_	(	self ):
     self.config_tester.run_common_tests()





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
        lowerCAmelCase_			=      type
        self.model_tester.create_and_check_model(*_lowerCamelCase )







  @slow
  def 						UpperCAmelCase_	(	self ):
     for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowerCAmelCase_			=      AlbertModel.from_pretrained(_lowerCamelCase )
        self.assertIsNotNone(_lowerCamelCase )






@require_torch
class     __UpperCAmelCase       ( unittest.TestCase		):





  @slow
  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      AlbertModel.from_pretrained('''albert-base-v2''' )
     lowerCAmelCase_			=      torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]] )
     lowerCAmelCase_			=      torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
     with torch.no_grad():
        lowerCAmelCase_			=      model(_lowerCamelCase    ,     attention_mask=_lowerCamelCase )[0]
     lowerCAmelCase_			=      torch.Size((1, 11, 768) )
     self.assertEqual(output.shape    ,     _lowerCamelCase )
     lowerCAmelCase_			=      torch.tensor(
         [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]] )

     self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,     _lowerCamelCase    ,     atol=1E-4 ) )






'''simple docstring'''

import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment



def snake_case_      (					)			->					str:
   lowerCAmelCase_			=      torch.nn.Linear(2	,   4)
   lowerCAmelCase_			=      torch.optim.AdamW(model.parameters()	,   lr=1.0)
   lowerCAmelCase_			=      torch.optim.lr_scheduler.OneCycleLR(__snake_case	,   max_lr=0.0_1	,   steps_per_epoch=2	,   epochs=1)
   lowerCAmelCase_			=      DataLoader(TensorDataset(torch.tensor([1, 2, 3])))
   lowerCAmelCase_			=      DataLoader(TensorDataset(torch.tensor([4, 5, 6])))

   return model, optimizer, scheduler, train_dl, valid_dl



def snake_case_      (					__snake_case  :							Any)			->					List[Any]:
   return (model.weight.abs().sum() + model.bias.abs().sum()).item()



def snake_case_      (					__snake_case  :							Optional[int])			->					Any:
   lowerCAmelCase_			=      torch.nn.Linear(*tuple(model.weight.T.shape)).state_dict()
   model.load_state_dict(__snake_case)






class     __UpperCAmelCase       ( __a		):





  @require_cuda
  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     assert PartialState._shared_state["_cpu"] is False
     assert PartialState._shared_state["device"].type == "cuda"
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      Accelerator(cpu=_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_			=      GradientState()
     assert state.num_steps == 1
     lowerCAmelCase_			=      4
     assert state.num_steps == 4

     assert state.sync_gradients is True
     lowerCAmelCase_			=      False
     assert state.sync_gradients is False
     GradientState._reset_state()





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()

     (
         (
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,
     )			=      accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     self.assertTrue(prepared_model in accelerator._models )
     self.assertTrue(prepared_optimizer in accelerator._optimizers )
     self.assertTrue(prepared_scheduler in accelerator._schedulers )
     self.assertTrue(prepared_train_dl in accelerator._dataloaders )
     self.assertTrue(prepared_valid_dl in accelerator._dataloaders )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     accelerator.free_memory()

     self.assertTrue(len(accelerator._models ) == 0 )
     self.assertTrue(len(accelerator._optimizers ) == 0 )
     self.assertTrue(len(accelerator._schedulers ) == 0 )
     self.assertTrue(len(accelerator._dataloaders ) == 0 )





  def 						UpperCAmelCase_	(	self ):
     PartialState._reset_state()

     # Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
     def noop(*_lowerCamelCase    ,     **_lowerCamelCase ):
        pass

     with patch('''torch.cuda.set_device'''    ,     _lowerCamelCase ), patch_environment(ACCELERATE_TORCH_DEVICE='''cuda:64''' ):
        lowerCAmelCase_			=      Accelerator()
        self.assertEqual(str(accelerator.state.device )    ,     '''cuda:64''' )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     lowerCAmelCase_			=      get_signature(_lowerCamelCase )

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # make sure loaded weights match
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     lowerCAmelCase_			=      get_signature(_lowerCamelCase )

     # saving hook
     def save_config(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
        lowerCAmelCase_			=      {'''class_name''': models[0].__class__.__name__}

        with open(os.path.join(_lowerCamelCase    ,     '''data.json''' )    ,     '''w''' ) as f:
           json.dump(_lowerCamelCase    ,     _lowerCamelCase )

        # loading hook
     def load_config(_lowerCamelCase    ,     _lowerCamelCase ):
        with open(os.path.join(_lowerCamelCase    ,     '''data.json''' )    ,     '''r''' ) as f:
           lowerCAmelCase_			=      json.load(_lowerCamelCase )

        lowerCAmelCase_			=      config['''class_name''']

     lowerCAmelCase_			=      accelerator.register_save_state_pre_hook(_lowerCamelCase )
     lowerCAmelCase_			=      accelerator.register_load_state_pre_hook(_lowerCamelCase )

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match with hooks
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # random class name to verify correct one is loaded
        lowerCAmelCase_			=      '''random'''

        # make sure loaded weights match with hooks
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )

        # mode.class_name is loaded from config
        self.assertTrue(model.class_name == model.__class__.__name__ )

     # remove hooks
     save_hook.remove()
     load_hook.remove()

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match with hooks removed
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # random class name to verify correct one is loaded
        lowerCAmelCase_			=      '''random'''

        # make sure loaded weights match with hooks removed
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )

        # mode.class_name is NOT loaded from config
        self.assertTrue(model.class_name != model.__class__.__name__ )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     lowerCAmelCase_			=      None

     # This should work
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      accelerator.prepare(
         _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     self.assertTrue(dummy_obj is None )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     lowerCAmelCase_			=      [1, 2, 3]

     # This should work
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      accelerator.prepare(
         _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Dummy object should have `_is_accelerate_prepared` set to `True`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Model is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Optimizer is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Scheduler is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )





  @slow
  @require_bnb
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map={'''''': 0}    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should work
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





  @slow
  @require_bnb
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      Accelerator()

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        model.tie_weights()
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      '''cpu'''

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     device_map=_lowerCamelCase    ,     load_in_abit=_lowerCamelCase    ,     llm_inta_enable_fpaa_cpu_offload=_lowerCamelCase )

     # This should not work and get value error
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





  @slow
  @require_bnb
  @require_multi_gpu
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      {'''distributed_type''': DistributedType.MULTI_GPU}

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        model.tie_weights()
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      1

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map=_lowerCamelCase    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should not work and get value error
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )

     PartialState._reset_state()





  @slow
  @require_bnb
  @require_multi_gpu
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      1

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map=_lowerCamelCase    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should work
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )







  @require_cuda
  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      torch.nn.Linear(10    ,     10 )
     lowerCAmelCase_			=      torch.optim.SGD(model.parameters()    ,     lr=0.01 )
     lowerCAmelCase_			=      Accelerator(cpu=_lowerCamelCase )
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





