



'''simple docstring'''





from collections import OrderedDict

from ...utils import logging
from .auto_factory import _BaseAutoModelClass, _LazyAutoMapping, auto_class_update
from .configuration_auto import CONFIG_MAPPING_NAMES


lowercase_									=    logging.get_logger(__name__)


lowercase_									=    OrderedDict(
    [
        # Base model mapping
        ("albert", "FlaxAlbertModel"),
        ("bart", "FlaxBartModel"),
        ("beit", "FlaxBeitModel"),
        ("bert", "FlaxBertModel"),
        ("big_bird", "FlaxBigBirdModel"),
        ("blenderbot", "FlaxBlenderbotModel"),
        ("blenderbot-small", "FlaxBlenderbotSmallModel"),
        ("clip", "FlaxCLIPModel"),
        ("distilbert", "FlaxDistilBertModel"),
        ("electra", "FlaxElectraModel"),
        ("gpt-sw3", "FlaxGPT2Model"),
        ("gpt2", "FlaxGPT2Model"),
        ("gpt_neo", "FlaxGPTNeoModel"),
        ("gptj", "FlaxGPTJModel"),
        ("longt5", "FlaxLongT5Model"),
        ("marian", "FlaxMarianModel"),
        ("mbart", "FlaxMBartModel"),
        ("mt5", "FlaxMT5Model"),
        ("opt", "FlaxOPTModel"),
        ("pegasus", "FlaxPegasusModel"),
        ("regnet", "FlaxRegNetModel"),
        ("resnet", "FlaxResNetModel"),
        ("roberta", "FlaxRobertaModel"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormModel"),
        ("roformer", "FlaxRoFormerModel"),
        ("t5", "FlaxT5Model"),
        ("vision-text-dual-encoder", "FlaxVisionTextDualEncoderModel"),
        ("vit", "FlaxViTModel"),
        ("wav2vec2", "FlaxWav2Vec2Model"),
        ("whisper", "FlaxWhisperModel"),
        ("xglm", "FlaxXGLMModel"),
        ("xlm-roberta", "FlaxXLMRobertaModel"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for pre-training mapping
        ("albert", "FlaxAlbertForPreTraining"),
        ("bart", "FlaxBartForConditionalGeneration"),
        ("bert", "FlaxBertForPreTraining"),
        ("big_bird", "FlaxBigBirdForPreTraining"),
        ("electra", "FlaxElectraForPreTraining"),
        ("longt5", "FlaxLongT5ForConditionalGeneration"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("mt5", "FlaxMT5ForConditionalGeneration"),
        ("roberta", "FlaxRobertaForMaskedLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMaskedLM"),
        ("roformer", "FlaxRoFormerForMaskedLM"),
        ("t5", "FlaxT5ForConditionalGeneration"),
        ("wav2vec2", "FlaxWav2Vec2ForPreTraining"),
        ("whisper", "FlaxWhisperForConditionalGeneration"),
        ("xlm-roberta", "FlaxXLMRobertaForMaskedLM"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Masked LM mapping
        ("albert", "FlaxAlbertForMaskedLM"),
        ("bart", "FlaxBartForConditionalGeneration"),
        ("bert", "FlaxBertForMaskedLM"),
        ("big_bird", "FlaxBigBirdForMaskedLM"),
        ("distilbert", "FlaxDistilBertForMaskedLM"),
        ("electra", "FlaxElectraForMaskedLM"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("roberta", "FlaxRobertaForMaskedLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMaskedLM"),
        ("roformer", "FlaxRoFormerForMaskedLM"),
        ("xlm-roberta", "FlaxXLMRobertaForMaskedLM"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Seq2Seq Causal LM mapping
        ("bart", "FlaxBartForConditionalGeneration"),
        ("blenderbot", "FlaxBlenderbotForConditionalGeneration"),
        ("blenderbot-small", "FlaxBlenderbotSmallForConditionalGeneration"),
        ("encoder-decoder", "FlaxEncoderDecoderModel"),
        ("longt5", "FlaxLongT5ForConditionalGeneration"),
        ("marian", "FlaxMarianMTModel"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("mt5", "FlaxMT5ForConditionalGeneration"),
        ("pegasus", "FlaxPegasusForConditionalGeneration"),
        ("t5", "FlaxT5ForConditionalGeneration"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Image-classsification
        ("beit", "FlaxBeitForImageClassification"),
        ("regnet", "FlaxRegNetForImageClassification"),
        ("resnet", "FlaxResNetForImageClassification"),
        ("vit", "FlaxViTForImageClassification"),
    ]
)

lowercase_									=    OrderedDict(
    [
        ("vision-encoder-decoder", "FlaxVisionEncoderDecoderModel"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Causal LM mapping
        ("bart", "FlaxBartForCausalLM"),
        ("bert", "FlaxBertForCausalLM"),
        ("big_bird", "FlaxBigBirdForCausalLM"),
        ("electra", "FlaxElectraForCausalLM"),
        ("gpt-sw3", "FlaxGPT2LMHeadModel"),
        ("gpt2", "FlaxGPT2LMHeadModel"),
        ("gpt_neo", "FlaxGPTNeoForCausalLM"),
        ("gptj", "FlaxGPTJForCausalLM"),
        ("opt", "FlaxOPTForCausalLM"),
        ("roberta", "FlaxRobertaForCausalLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForCausalLM"),
        ("xglm", "FlaxXGLMForCausalLM"),
        ("xlm-roberta", "FlaxXLMRobertaForCausalLM"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Sequence Classification mapping
        ("albert", "FlaxAlbertForSequenceClassification"),
        ("bart", "FlaxBartForSequenceClassification"),
        ("bert", "FlaxBertForSequenceClassification"),
        ("big_bird", "FlaxBigBirdForSequenceClassification"),
        ("distilbert", "FlaxDistilBertForSequenceClassification"),
        ("electra", "FlaxElectraForSequenceClassification"),
        ("mbart", "FlaxMBartForSequenceClassification"),
        ("roberta", "FlaxRobertaForSequenceClassification"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForSequenceClassification"),
        ("roformer", "FlaxRoFormerForSequenceClassification"),
        ("xlm-roberta", "FlaxXLMRobertaForSequenceClassification"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Question Answering mapping
        ("albert", "FlaxAlbertForQuestionAnswering"),
        ("bart", "FlaxBartForQuestionAnswering"),
        ("bert", "FlaxBertForQuestionAnswering"),
        ("big_bird", "FlaxBigBirdForQuestionAnswering"),
        ("distilbert", "FlaxDistilBertForQuestionAnswering"),
        ("electra", "FlaxElectraForQuestionAnswering"),
        ("mbart", "FlaxMBartForQuestionAnswering"),
        ("roberta", "FlaxRobertaForQuestionAnswering"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForQuestionAnswering"),
        ("roformer", "FlaxRoFormerForQuestionAnswering"),
        ("xlm-roberta", "FlaxXLMRobertaForQuestionAnswering"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Token Classification mapping
        ("albert", "FlaxAlbertForTokenClassification"),
        ("bert", "FlaxBertForTokenClassification"),
        ("big_bird", "FlaxBigBirdForTokenClassification"),
        ("distilbert", "FlaxDistilBertForTokenClassification"),
        ("electra", "FlaxElectraForTokenClassification"),
        ("roberta", "FlaxRobertaForTokenClassification"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForTokenClassification"),
        ("roformer", "FlaxRoFormerForTokenClassification"),
        ("xlm-roberta", "FlaxXLMRobertaForTokenClassification"),
    ]
)

lowercase_									=    OrderedDict(
    [
        # Model for Multiple Choice mapping
        ("albert", "FlaxAlbertForMultipleChoice"),
        ("bert", "FlaxBertForMultipleChoice"),
        ("big_bird", "FlaxBigBirdForMultipleChoice"),
        ("distilbert", "FlaxDistilBertForMultipleChoice"),
        ("electra", "FlaxElectraForMultipleChoice"),
        ("roberta", "FlaxRobertaForMultipleChoice"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMultipleChoice"),
        ("roformer", "FlaxRoFormerForMultipleChoice"),
        ("xlm-roberta", "FlaxXLMRobertaForMultipleChoice"),
    ]
)

lowercase_									=    OrderedDict(
    [
        ("bert", "FlaxBertForNextSentencePrediction"),
    ]
)

lowercase_									=    OrderedDict(
    [
        ("speech-encoder-decoder", "FlaxSpeechEncoderDecoderModel"),
        ("whisper", "FlaxWhisperForConditionalGeneration"),
    ]
)

lowercase_									=    OrderedDict(
    [
        ("whisper", "FlaxWhisperForAudioClassification"),
    ]
)

lowercase_									=    _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_MAPPING_NAMES)
lowercase_									=    _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_PRETRAINING_MAPPING_NAMES)
lowercase_									=    _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MASKED_LM_MAPPING_NAMES)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES)
lowercase_									=    _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES
)
lowercase_									=    _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Dict		=			FLAX_MODEL_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModel)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			List[Any]		=			FLAX_MODEL_FOR_PRETRAINING_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForPreTraining, head_doc="pretraining")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			int		=			FLAX_MODEL_FOR_CAUSAL_LM_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForCausalLM, head_doc="causal language modeling")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[Any]		=			FLAX_MODEL_FOR_MASKED_LM_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForMaskedLM, head_doc="masked language modeling")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[Any]		=			FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForSeqaSeqLM, head_doc="sequence-to-sequence language modeling", checkpoint_for_example="t5-base"
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Any		=			FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForSequenceClassification, head_doc="sequence classification"
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[int]		=			FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForQuestionAnswering, head_doc="question answering")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[int]		=			FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForTokenClassification, head_doc="token classification"
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			str		=			FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForMultipleChoice, head_doc="multiple choice")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Tuple		=			FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForNextSentencePrediction, head_doc="next sentence prediction"
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			str		=			FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForImageClassification, head_doc="image classification"
)






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[int]		=			FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING





lowercase_									=    auto_class_update(FlaxAutoModelForVisionaSeq, head_doc="vision-to-text modeling")






class __A     (    _BaseAutoModelClass		):



 '''simple docstring'''







 __lowerCamelCase       :			Any		=			FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING





lowercase_									=    auto_class_update(
    FlaxAutoModelForSpeechSeqaSeq, head_doc="sequence-to-sequence speech-to-text modeling"
)








'''simple docstring'''





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class __A     (    nn.Module		):



 '''simple docstring'''







 def __init__(self			)       ->	Dict:


      """simple docstring"""



      super().__init__()
      _a           =    nn.Linear(3					,  4			)
      _a           =    nn.BatchNormad(4			)
      _a           =    nn.Linear(4					,  5			)



 def    a__    (self					,  A			)       ->	Dict:


      """simple docstring"""



      return self.lineara(self.batchnorm(self.lineara(A			)			)			)






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  *A					,  **A			)       ->	Optional[Any]:


      """simple docstring"""



      return (args[0] + 1,) + args[1:], kwargs






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  A			)       ->	int:


      """simple docstring"""



      return output + 1






class __A     (    unittest.TestCase		):



 '''simple docstring'''







 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      self.assertEqual(test_model._hf_hook					,  A			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      add_hook_to_module(A					,  A					,  append=A			)

      self.assertEqual(isinstance(test_model._hf_hook					,  A			)					,  A			)
      self.assertEqual(len(test_model._hf_hook.hooks			)					,  2			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(x + 1			)
      _a           =    test_model(x + 2			)

      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PreForwardHook()					,  PreForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  A					,  atol=1E-5			)



 def    a__    (self			)       ->	str:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PostForwardHook()					,  PostForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  output + 2					,  atol=1E-5			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1			)			)
      self.assertTrue(outputa.requires_grad			)

      _a           =    True
      _a           =    test_model(A			)
      self.assertFalse(outputa.requires_grad			)



 @require_multi_gpu
 def    a__    (self			)       ->	List[Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=1			)			)

      self.assertEqual(model.lineara.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device(0			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device(1			)			)

      # We can still make a forward pass. The input does not need to be on any particular device
      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(1			)			)

      # We can add a general hook to put back output on same device as input.
      add_hook_to_module(A					,  AlignDevicesHook(io_same_device=A			)			)
      _a           =    torch.randn(2					,  3			).to(0			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(0			)			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    {'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(hook_kwargs['''execution_device''']			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      _a           =    {
          '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
          '''offload''': True,
          '''offload_buffers''': True,
      }

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(A					,  execution_device=A					,  offload=A			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(A					,  execution_device=A					,  offload=A					,  offload_buffers=A			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()					,  offload_buffers=A					,  )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



