



'''simple docstring'''





import os
import re
import shutil
import sys
import tempfile
import unittest

import black


lowercase_									=    os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
lowercase_									=    "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"






class __A     (    unittest.TestCase		):



 '''simple docstring'''







 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      _a           =    tempfile.mkdtemp()
      os.makedirs(os.path.join(self.transformer_dir					,  '''models/bert/'''			)			)
      _a           =    self.transformer_dir
      shutil.copy(
          os.path.join(A					,  '''src/transformers/models/bert/modeling_bert.py'''			)					,  os.path.join(self.transformer_dir					,  '''models/bert/modeling_bert.py'''			)					,  )



 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    '''src/transformers'''
      shutil.rmtree(self.transformer_dir			)



 def    a__    (self					,  A					,  A					,  A					,  A=None			)       ->	Dict:


      """simple docstring"""



      _a           =    comment + f'''\nclass {class_name}(nn.Module):\n''' + class_code
      if overwrite_result is not None:
           _a           =    comment + f'''\nclass {class_name}(nn.Module):\n''' + overwrite_result
      _a           =    black.Mode(target_versions={black.TargetVersion.PYaa}					,  line_length=119			)
      _a           =    black.format_str(A					,  mode=A			)
      _a           =    os.path.join(self.transformer_dir					,  '''new_code.py'''			)
      with open(A					,  '''w'''					,  newline='''\n'''			) as f:
           f.write(A			)
      if overwrite_result is None:
           self.assertTrue(len(check_copies.is_copy_consistent(A			)			) == 0			)
      else:
           check_copies.is_copy_consistent(f.name					,  overwrite=A			)
           with open(A					,  '''r'''			) as f:
                self.assertTrue(f.read()					,  A			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    check_copies.find_code_in_transformers('''models.bert.modeling_bert.BertLMPredictionHead'''			)
      self.assertEqual(A					,  A			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      self.check_copy_consistency(
          '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'''					,  '''BertLMPredictionHead'''					,  REFERENCE_CODE + '''\n'''					,  )

      # With no empty line at the end
      self.check_copy_consistency(
          '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'''					,  '''BertLMPredictionHead'''					,  A					,  )

      # Copy consistency with rename
      self.check_copy_consistency(
          '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'''					,  '''TestModelLMPredictionHead'''					,  re.sub('''Bert'''					,  '''TestModel'''					,  A			)					,  )

      # Copy consistency with a really long name
      _a           =    '''TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason'''
      self.check_copy_consistency(
          f'''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'''					,  f'''{long_class_name}LMPredictionHead'''					,  re.sub('''Bert'''					,  A					,  A			)					,  )

      # Copy consistency with overwrite
      self.check_copy_consistency(
          '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'''					,  '''TestModelLMPredictionHead'''					,  A					,  overwrite_result=re.sub('''Bert'''					,  '''TestModel'''					,  A			)					,  )



 def    a__    (self			)       ->	int:


      """simple docstring"""



      _a           =    check_copies.LOCALIZED_READMES['''README_zh-hans.md''']

      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'''
          ''' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'''
          ''' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'''
          ''' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1.'''
          ''' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),'''
          ''' released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'''
          ''' lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same'''
          ''' method has been applied to compress GPT2 into'''
          ''' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'''
          ''' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'''
          ''' Multilingual BERT into'''
          ''' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'''
          ''' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**'''
          ''' (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders'''
          ''' as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang'''
          ''' Luong, Quoc V. Le, Christopher D. Manning.'''
      )
      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'''
          ''' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'''
          ''' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'''
          ''' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'''
      )
      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'''
          ''' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'''
          ''' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'''
          ''' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1.'''
          ''' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文'''
          ''' [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'''
          ''' lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same'''
          ''' method has been applied to compress GPT2 into'''
          ''' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'''
          ''' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'''
          ''' Multilingual BERT into'''
          ''' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'''
          ''' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自'''
          ''' Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather'''
          ''' than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,'''
          ''' Christopher D. Manning 发布。\n'''
      )

      _a    ,       _a           =    check_copies.convert_to_localized_md(
          A					,  A					,  localized_readme['''format_model_list''']			)

      self.assertFalse(A			)
      self.assertEqual(A					,  A			)

      _a    ,       _a           =    check_copies.convert_to_localized_md(
          A					,  A					,  localized_readme['''format_model_list''']			)

      # Check whether the number of models is equal to README.md after conversion.
      self.assertTrue(A			)

      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'''
          ''' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'''
          ''' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'''
          ''' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.'''
      )
      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and'''
          ''' the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'''
          ''' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'''
          ''' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'''
      )
      _a           =    (
          '''1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'''
          ''' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'''
          ''' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'''
          ''' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'''
      )

      _a    ,       _a           =    check_copies.convert_to_localized_md(
          A					,  A					,  localized_readme['''format_model_list''']			)

      # Check if the model link is synchronized.
      self.assertEqual(A					,  A			)








'''simple docstring'''





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class __A     (    nn.Module		):



 '''simple docstring'''







 def __init__(self			)       ->	Dict:


      """simple docstring"""



      super().__init__()
      _a           =    nn.Linear(3					,  4			)
      _a           =    nn.BatchNormad(4			)
      _a           =    nn.Linear(4					,  5			)



 def    a__    (self					,  A			)       ->	Dict:


      """simple docstring"""



      return self.lineara(self.batchnorm(self.lineara(A			)			)			)






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  *A					,  **A			)       ->	Optional[Any]:


      """simple docstring"""



      return (args[0] + 1,) + args[1:], kwargs






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  A			)       ->	int:


      """simple docstring"""



      return output + 1






class __A     (    unittest.TestCase		):



 '''simple docstring'''







 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      self.assertEqual(test_model._hf_hook					,  A			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      add_hook_to_module(A					,  A					,  append=A			)

      self.assertEqual(isinstance(test_model._hf_hook					,  A			)					,  A			)
      self.assertEqual(len(test_model._hf_hook.hooks			)					,  2			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(x + 1			)
      _a           =    test_model(x + 2			)

      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PreForwardHook()					,  PreForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  A					,  atol=1E-5			)



 def    a__    (self			)       ->	str:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PostForwardHook()					,  PostForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  output + 2					,  atol=1E-5			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1			)			)
      self.assertTrue(outputa.requires_grad			)

      _a           =    True
      _a           =    test_model(A			)
      self.assertFalse(outputa.requires_grad			)



 @require_multi_gpu
 def    a__    (self			)       ->	List[Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=1			)			)

      self.assertEqual(model.lineara.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device(0			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device(1			)			)

      # We can still make a forward pass. The input does not need to be on any particular device
      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(1			)			)

      # We can add a general hook to put back output on same device as input.
      add_hook_to_module(A					,  AlignDevicesHook(io_same_device=A			)			)
      _a           =    torch.randn(2					,  3			).to(0			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(0			)			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    {'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(hook_kwargs['''execution_device''']			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      _a           =    {
          '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
          '''offload''': True,
          '''offload_buffers''': True,
      }

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(A					,  execution_device=A					,  offload=A			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(A					,  execution_device=A					,  offload=A					,  offload_buffers=A			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()					,  offload_buffers=A					,  )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



