
"""simple docstring"""

from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
   import PIL

SCREAMING_SNAKE_CASE_    					=						logging.get_logger(__name__)




def     lowercase__						(					lowerCAmelCase				:					int ) ->      List[List[ImageInput]]:






    """simple docstring"""




    if isinstance(lowerCAmelCase ,				(list, tuple) ) and isinstance(videos[0] ,				(list, tuple) ) and is_valid_image(videos[0][0] ):
        return videos

    elif isinstance(lowerCAmelCase ,				(list, tuple) ) and is_valid_image(videos[0] ):
        return [videos]

    elif is_valid_image(lowerCAmelCase ):
        return [[videos]]

    raise ValueError(F"Could not make batched video from {videos}" )







class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):
      __SCREAMING_SNAKE_CASE   :							Union[str, Any]							      =	["pixel_values"]




      def __init__(							self   ,       lowercase_ = True   ,       lowercase_ = None   ,       lowercase_ = PILImageResampling.BILINEAR   ,       lowercase_ = True   ,       lowercase_ = None   ,       lowercase_ = True   ,       lowercase_ = 1 / 2_5_5   ,       lowercase_ = True   ,       lowercase_ = True   ,       lowercase_ = None   ,       lowercase_ = None   ,       **lowercase_   ,       )						->					None:
          super().__init__(**lowercase_							)
          UpperCAmelCase									=					size if size is not None else {'shortest_edge': 2_5_6}
          UpperCAmelCase									=					get_size_dict(lowercase_   ,       default_to_square=lowercase_							)
          UpperCAmelCase									=					crop_size if crop_size is not None else {'height': 2_2_4, 'width': 2_2_4}
          UpperCAmelCase									=					get_size_dict(lowercase_   ,       param_name='crop_size'							)

          UpperCAmelCase									=					do_resize
          UpperCAmelCase									=					size
          UpperCAmelCase									=					do_center_crop
          UpperCAmelCase									=					crop_size
          UpperCAmelCase									=					resample
          UpperCAmelCase									=					do_rescale
          UpperCAmelCase									=					rescale_factor
          UpperCAmelCase									=					offset
          UpperCAmelCase									=					do_normalize
          UpperCAmelCase									=					image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
          UpperCAmelCase									=					image_std if image_std is not None else IMAGENET_STANDARD_STD




      def     a_      (							self   ,       lowercase_   ,       lowercase_   ,       lowercase_ = PILImageResampling.BILINEAR   ,       lowercase_ = None   ,       **lowercase_   ,       )						->					np.ndarray:
          UpperCAmelCase									=					get_size_dict(lowercase_   ,       default_to_square=lowercase_							)
          if "shortest_edge" in size:
              UpperCAmelCase									=					get_resize_output_image_size(lowercase_   ,       size['shortest_edge']   ,       default_to_square=lowercase_							)
          elif "height" in size and "width" in size:
              UpperCAmelCase									=					(size['height'], size['width'])
          else:
              raise ValueError(F"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"							)
          return resize(lowercase_   ,       size=lowercase_   ,       resample=lowercase_   ,       data_format=lowercase_   ,       **lowercase_							)




      def     a_      (							self   ,       lowercase_   ,       lowercase_   ,       lowercase_ = None   ,       **lowercase_   ,       )						->					np.ndarray:
          UpperCAmelCase									=					get_size_dict(lowercase_							)
          if "height" not in size or "width" not in size:
              raise ValueError(F"Size must have 'height' and 'width' as keys. Got {size.keys()}"							)
          return center_crop(lowercase_   ,       size=(size['height'], size['width'])   ,       data_format=lowercase_   ,       **lowercase_							)




      def     a_      (							self   ,       lowercase_   ,       lowercase_   ,       lowercase_ = True   ,       lowercase_ = None   ,       **lowercase_   ,       )						->					str:
          UpperCAmelCase									=					image.astype(np.floataa							)
          if offset:
              UpperCAmelCase									=					image - (scale / 2)
          return rescale(lowercase_   ,       scale=lowercase_   ,       data_format=lowercase_   ,       **lowercase_							)




      def     a_      (							self   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_ = None   ,       **lowercase_   ,       )						->					np.ndarray:
          return normalize(lowercase_   ,       mean=lowercase_   ,       std=lowercase_   ,       data_format=lowercase_   ,       **lowercase_							)




      def     a_      (							self   ,       lowercase_   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = ChannelDimension.FIRST   ,       )						->					np.ndarray:
          if do_resize and size is None or resample is None:
              raise ValueError('Size and resample must be specified if do_resize is True.'							)

          if do_center_crop and crop_size is None:
              raise ValueError('Crop size must be specified if do_center_crop is True.'							)

          if do_rescale and rescale_factor is None:
              raise ValueError('Rescale factor must be specified if do_rescale is True.'							)

          if do_normalize and (image_mean is None or image_std is None):
              raise ValueError('Image mean and std must be specified if do_normalize is True.'							)

          if offset and not do_rescale:
              raise ValueError('For offset, do_rescale must also be set to True.'							)

          # All transformations expect numpy arrays.
          UpperCAmelCase									=					to_numpy_array(lowercase_							)

          if do_resize:
              UpperCAmelCase									=					self.resize(image=lowercase_   ,       size=lowercase_   ,       resample=lowercase_							)

          if do_center_crop:
              UpperCAmelCase									=					self.center_crop(lowercase_   ,       size=lowercase_							)

          if do_rescale:
              UpperCAmelCase									=					self.rescale(image=lowercase_   ,       scale=lowercase_   ,       offset=lowercase_							)

          if do_normalize:
              UpperCAmelCase									=					self.normalize(image=lowercase_   ,       mean=lowercase_   ,       std=lowercase_							)

          UpperCAmelCase									=					to_channel_dimension_format(lowercase_   ,       lowercase_							)
          return image




      def     a_      (							self   ,       lowercase_   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = None   ,       lowercase_ = ChannelDimension.FIRST   ,       **lowercase_   ,       )						->					PIL.Image.Image:
          UpperCAmelCase									=					do_resize if do_resize is not None else self.do_resize
          UpperCAmelCase									=					resample if resample is not None else self.resample
          UpperCAmelCase									=					do_center_crop if do_center_crop is not None else self.do_center_crop
          UpperCAmelCase									=					do_rescale if do_rescale is not None else self.do_rescale
          UpperCAmelCase									=					rescale_factor if rescale_factor is not None else self.rescale_factor
          UpperCAmelCase									=					offset if offset is not None else self.offset
          UpperCAmelCase									=					do_normalize if do_normalize is not None else self.do_normalize
          UpperCAmelCase									=					image_mean if image_mean is not None else self.image_mean
          UpperCAmelCase									=					image_std if image_std is not None else self.image_std

          UpperCAmelCase									=					size if size is not None else self.size
          UpperCAmelCase									=					get_size_dict(lowercase_   ,       default_to_square=lowercase_							)
          UpperCAmelCase									=					crop_size if crop_size is not None else self.crop_size
          UpperCAmelCase									=					get_size_dict(lowercase_   ,       param_name='crop_size'							)

          if not valid_images(lowercase_							):
              raise ValueError(
                  'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
                  'torch.Tensor, tf.Tensor or jax.ndarray.'							)

          UpperCAmelCase									=					make_batched(lowercase_							)

          UpperCAmelCase									=					[
              [
                  self._preprocess_image(
                      image=lowercase_   ,       do_resize=lowercase_   ,       size=lowercase_   ,       resample=lowercase_   ,       do_center_crop=lowercase_   ,       crop_size=lowercase_   ,       do_rescale=lowercase_   ,       rescale_factor=lowercase_   ,       offset=lowercase_   ,       do_normalize=lowercase_   ,       image_mean=lowercase_   ,       image_std=lowercase_   ,       data_format=lowercase_   ,       )
                  for img in video
              ]
              for video in videos
          ]

          UpperCAmelCase									=					{'pixel_values': videos}
          return BatchFeature(data=lowercase_   ,       tensor_type=lowercase_							)





"""simple docstring"""

import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu
class      _UpperCAmelCase							(		nn.Module							):




      def __init__(							self							)						->					Any:
          super().__init__()
          UpperCAmelCase									=					nn.Linear(3   ,       4							)
          UpperCAmelCase									=					nn.BatchNormad(4							)
          UpperCAmelCase									=					nn.Linear(4   ,       5							)




      def     a_      (							self   ,       lowercase_							)						->					Any:
          return self.lineara(self.batchnorm(self.lineara(lowercase_							)							)							)

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       *lowercase_   ,       **lowercase_							)						->					List[str]:
          return (args[0] + 1,) + args[1:], kwargs

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       lowercase_							)						->					Union[str, Any]:
          return output + 1

class      _UpperCAmelCase							(		unittest.TestCase							):




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          self.assertEqual(test_model._hf_hook   ,       lowercase_							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          add_hook_to_module(lowercase_   ,       lowercase_   ,       append=lowercase_							)

          self.assertEqual(isinstance(test_model._hf_hook   ,       lowercase_							)   ,       lowercase_							)
          self.assertEqual(len(test_model._hf_hook.hooks							)   ,       2							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(x + 1							)
          UpperCAmelCase									=					test_model(x + 2							)

          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PreForwardHook()   ,       PreForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)




      def     a_      (							self							)						->					List[str]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PostForwardHook()   ,       PostForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       output + 2   ,       atol=1E-5							)




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1							)							)
          self.assertTrue(outputa.requires_grad							)

          UpperCAmelCase									=					True
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertFalse(outputa.requires_grad							)




      @require_multi_gpu
      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=1							)							)

          self.assertEqual(model.lineara.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device(0							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device(1							)							)

          # We can still make a forward pass. The input does not need to be on any particular device
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(1							)							)

          # We can add a general hook to put back output on same device as input.
          add_hook_to_module(lowercase_   ,       AlignDevicesHook(io_same_device=lowercase_							)							)
          UpperCAmelCase									=					torch.randn(2   ,       3							).to(0							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(0							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(hook_kwargs['execution_device']							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          UpperCAmelCase									=					{
              'execution_device': 0 if torch.cuda.is_available() else 'cpu',
              'offload': True,
              'offload_buffers': True,
          }

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       offload_buffers=lowercase_							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()   ,       offload_buffers=lowercase_   ,       )

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)



