
'''simple docstring'''


import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu


class 						lowercase__	(  nn.Module   ):






   '''simple docstring'''







   def __init__(			self   ):

      '''simple docstring'''







      super().__init__()
      UpperCamelCase       =  nn.Linear(3 ,   4   )
      UpperCamelCase       =  nn.BatchNormad(4   )
      UpperCamelCase       =  nn.Linear(4 ,   5   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.lineara(self.batchnorm(self.lineara(lowerCamelCase__   )   )   )




class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      return (args[0] + 1,) + args[1:], kwargs




class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      return output + 1




class 						lowercase__	(  unittest.TestCase   ):






   '''simple docstring'''







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      UpperCamelCase       =  ModelHook()

      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      self.assertEqual(test_model._hf_hook ,   lowerCamelCase__   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''_old_forward'''   )   )

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__ ,   '''forward'''   )
      self.assertListEqual(list(inspect.signature(test_model.forward   ).parameters   ) ,   ['''x''']   )

      remove_hook_from_module(lowerCamelCase__   )
      self.assertFalse(hasattr(lowerCamelCase__ ,   '''_hf_hook'''   )   )
      self.assertFalse(hasattr(lowerCamelCase__ ,   '''_old_forward'''   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      UpperCamelCase       =  ModelHook()

      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__ ,   append=lowerCamelCase__   )

      self.assertEqual(isinstance(test_model._hf_hook ,   lowerCamelCase__   ) ,   lowerCamelCase__   )
      self.assertEqual(len(test_model._hf_hook.hooks   ) ,   2   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''_old_forward'''   )   )

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__ ,   '''forward'''   )
      self.assertListEqual(list(inspect.signature(test_model.forward   ).parameters   ) ,   ['''x''']   )

      remove_hook_from_module(lowerCamelCase__   )
      self.assertFalse(hasattr(lowerCamelCase__ ,   '''_hf_hook'''   )   )
      self.assertFalse(hasattr(lowerCamelCase__ ,   '''_old_forward'''   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  test_model(x + 1   )
      UpperCamelCase       =  test_model(x + 2   )

      UpperCamelCase       =  PreForwardHook()
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertTrue(torch.allclose(lowerCamelCase__ ,   lowerCamelCase__ ,   atol=1e-5   )   )

      # Attaching a hook to a model when it already has one replaces, does not chain
      UpperCamelCase       =  PreForwardHook()
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertTrue(torch.allclose(lowerCamelCase__ ,   lowerCamelCase__ ,   atol=1e-5   )   )

      # You need to use the sequential hook to chain two or more hooks
      UpperCamelCase       =  SequentialHook(PreForwardHook() ,   PreForwardHook()   )
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )

      UpperCamelCase       =  test_model(lowerCamelCase__   )
      assert torch.allclose(lowerCamelCase__ ,   lowerCamelCase__ ,   atol=1e-5   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )

      UpperCamelCase       =  PostForwardHook()
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertTrue(torch.allclose(lowerCamelCase__ ,   output + 1 ,   atol=1e-5   )   )

      # Attaching a hook to a model when it already has one replaces, does not chain
      UpperCamelCase       =  PostForwardHook()
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertTrue(torch.allclose(lowerCamelCase__ ,   output + 1 ,   atol=1e-5   )   )

      # You need to use the sequential hook to chain two or more hooks
      UpperCamelCase       =  SequentialHook(PostForwardHook() ,   PostForwardHook()   )
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )

      UpperCamelCase       =  test_model(lowerCamelCase__   )
      assert torch.allclose(lowerCamelCase__ ,   output + 2 ,   atol=1e-5   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )

      UpperCamelCase       =  PostForwardHook()
      add_hook_to_module(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertTrue(torch.allclose(lowerCamelCase__ ,   output + 1   )   )
      self.assertTrue(outputa.requires_grad   )

      UpperCamelCase       =  True
      UpperCamelCase       =  test_model(lowerCamelCase__   )
      self.assertFalse(outputa.requires_grad   )







   @require_multi_gpu
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()
      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # This will move each submodule on different devices
      add_hook_to_module(model.lineara ,   AlignDevicesHook(execution_device=0   )   )
      add_hook_to_module(model.batchnorm ,   AlignDevicesHook(execution_device=0   )   )
      add_hook_to_module(model.lineara ,   AlignDevicesHook(execution_device=1   )   )

      self.assertEqual(model.lineara.weight.device ,   torch.device(0   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device(0   )   )
      self.assertEqual(model.batchnorm.running_mean.device ,   torch.device(0   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device(1   )   )

      # We can still make a forward pass. The input does not need to be on any particular device
      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   torch.device(1   )   )

      # We can add a general hook to put back output on same device as input.
      add_hook_to_module(lowerCamelCase__ ,   AlignDevicesHook(io_same_device=lowerCamelCase__   )   )
      UpperCamelCase       =  torch.randn(2 ,   3   ).to(0   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   torch.device(0   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # This will move each submodule on different devices
      UpperCamelCase       =  {'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

      add_hook_to_module(model.lineara ,   AlignDevicesHook(**lowerCamelCase__   )   )
      add_hook_to_module(model.batchnorm ,   AlignDevicesHook(**lowerCamelCase__   )   )
      add_hook_to_module(model.lineara ,   AlignDevicesHook(**lowerCamelCase__   )   )

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      # Buffers are not included in the offload by default, so are on the execution device
      UpperCamelCase       =  torch.device(hook_kwargs['''execution_device''']   )
      self.assertEqual(model.batchnorm.running_mean.device ,   lowerCamelCase__   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara   )
      remove_hook_from_module(model.batchnorm   )
      remove_hook_from_module(model.lineara   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # Now test with buffers included in the offload
      UpperCamelCase       =  {
          '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
          '''offload''': True,
          '''offload_buffers''': True,
      }

      add_hook_to_module(model.lineara ,   AlignDevicesHook(**lowerCamelCase__   )   )
      add_hook_to_module(model.batchnorm ,   AlignDevicesHook(**lowerCamelCase__   )   )
      add_hook_to_module(model.lineara ,   AlignDevicesHook(**lowerCamelCase__   )   )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.running_mean.device ,   torch.device('''meta'''   )   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara   )
      remove_hook_from_module(model.batchnorm   )
      remove_hook_from_module(model.lineara   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # This will move each submodule on different devices
      UpperCamelCase       =  0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(lowerCamelCase__ ,   execution_device=lowerCamelCase__ ,   offload=lowerCamelCase__   )

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      # Buffers are not included in the offload by default, so are on the execution device
      UpperCamelCase       =  torch.device(lowerCamelCase__   )
      self.assertEqual(model.batchnorm.running_mean.device ,   lowerCamelCase__   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(lowerCamelCase__   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # Now test with buffers included in the offload
      attach_align_device_hook(lowerCamelCase__ ,   execution_device=lowerCamelCase__ ,   offload=lowerCamelCase__ ,   offload_buffers=lowerCamelCase__   )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.running_mean.device ,   torch.device('''meta'''   )   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(lowerCamelCase__   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # This will move each submodule on different devices
      UpperCamelCase       =  0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(
          lowerCamelCase__ ,   execution_device=lowerCamelCase__ ,   offload=lowerCamelCase__ ,   weights_map=model.state_dict()   )

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      # Buffers are not included in the offload by default, so are on the execution device
      UpperCamelCase       =  torch.device(lowerCamelCase__   )
      self.assertEqual(model.batchnorm.running_mean.device ,   lowerCamelCase__   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(lowerCamelCase__   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )

      # Now test with buffers included in the offload
      attach_align_device_hook(
          lowerCamelCase__ ,   execution_device=lowerCamelCase__ ,   offload=lowerCamelCase__ ,   weights_map=model.state_dict() ,   offload_buffers=lowerCamelCase__ ,   )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''meta'''   )   )
      self.assertEqual(model.batchnorm.running_mean.device ,   torch.device('''meta'''   )   )

      UpperCamelCase       =  torch.randn(2 ,   3   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.assertEqual(output.device ,   lowerCamelCase__   )

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(lowerCamelCase__   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.batchnorm.weight.device ,   torch.device('''cpu'''   )   )
      self.assertEqual(model.lineara.weight.device ,   torch.device('''cpu'''   )   )







'''simple docstring'''


def  __snake_case     (    _UpperCAmelCase					:       list[list[float]]):
   UpperCamelCase       =  []
   for data in source_data:
      for i, el in enumerate(_UpperCAmelCase):
         if len(_UpperCAmelCase) < i + 1:
            data_lists.append([])
         data_lists[i].append(float(_UpperCAmelCase))
   return data_lists



def  __snake_case     (    _UpperCAmelCase					:       list[list[float]],      _UpperCAmelCase					:       list[int]):
   UpperCamelCase       =  []
   for dlist, weight in zip(_UpperCAmelCase,      _UpperCAmelCase):
      UpperCamelCase       =  min(_UpperCAmelCase)
      UpperCamelCase       =  max(_UpperCAmelCase)

      UpperCamelCase       =  []
      # for weight 0 score is 1 - actual score
      if weight == 0:
         for item in dlist:
            try:
               score.append(1 - ((item - mind) / (maxd - mind)))
            except ZeroDivisionError:
               score.append(1)

      elif weight == 1:
         for item in dlist:
            try:
               score.append((item - mind) / (maxd - mind))
            except ZeroDivisionError:
               score.append(0)

        # weight not 0 or 1
      else:
         UpperCamelCase       =  f'Invalid weight of {weight:f} provided'
         raise ValueError(_UpperCAmelCase)

      score_lists.append(_UpperCAmelCase)

   return score_lists



def  __snake_case     (    _UpperCAmelCase					:       list[list[float]]):
   UpperCamelCase       =  [0 for i in range(len(score_lists[0]))]

   for slist in score_lists:
      for j, ele in enumerate(_UpperCAmelCase):
         UpperCamelCase       =  final_scores[j] + ele

   return final_scores



def  __snake_case     (    _UpperCAmelCase					:       list[list[float]],      _UpperCAmelCase					:       list[int]):
   UpperCamelCase       =  get_data(_UpperCAmelCase)
   UpperCamelCase       =  calculate_each_score(_UpperCAmelCase,      _UpperCAmelCase)
   UpperCamelCase       =  generate_final_scores(_UpperCAmelCase)

   # append scores to source data
   for i, ele in enumerate(_UpperCAmelCase):
      source_data[i].append(_UpperCAmelCase)

   return source_data





