





"""simple docstring"""





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class 							snake_case							(			__lowerCAmelCase ):

						'''simple docstring'''



						def __init__(							self       :      Any						,				__lowercase       :      List[Any]						,				__lowercase       :      str = None						,				__lowercase       :      Dict = None						,				__lowercase       :      Optional[Any] = None						,				__lowercase       :      str = False						,				__lowercase       :      Any = False						,				__lowercase       :      Tuple = None						,				__lowercase       :      Dict = None						,				**__lowercase       :      Tuple						,				):


								'''simple docstring'''




								super().__init__(
								    lowerCAmelCase_						,				split=lowerCAmelCase_						,				features=lowerCAmelCase_						,				cache_dir=lowerCAmelCase_						,				keep_in_memory=lowerCAmelCase_						,				streaming=lowerCAmelCase_						,				num_proc=lowerCAmelCase_						,				**lowerCAmelCase_						,				)
								__UpperCAmelCase    :						Union[str, Any]       =						field
								__UpperCAmelCase    :						str       =						path_or_paths if isinstance(lowerCAmelCase_						,				lowerCAmelCase_ ) else {self.split: path_or_paths}
								__UpperCAmelCase    :						List[str]       =						Json(
								    cache_dir=lowerCAmelCase_						,				data_files=lowerCAmelCase_						,				features=lowerCAmelCase_						,				field=lowerCAmelCase_						,				**lowerCAmelCase_						,				)







						def       A_					(							self       :      Union[str, Any] ):


								'''simple docstring'''




								if self.streaming:
										__UpperCAmelCase    :						Tuple       =						self.builder.as_streaming_dataset(split=self.split )
								# Build regular (map-style) dataset
								else:
										__UpperCAmelCase    :						str       =						None
										__UpperCAmelCase    :						Optional[Any]       =						None
										__UpperCAmelCase    :						Dict       =						None
										__UpperCAmelCase    :						List[str]       =						None

										self.builder.download_and_prepare(
										    download_config=lowerCAmelCase_						,				download_mode=lowerCAmelCase_						,				verification_mode=lowerCAmelCase_						,				base_path=lowerCAmelCase_						,				num_proc=self.num_proc						,				)
										__UpperCAmelCase    :						Any       =						self.builder.as_dataset(
										    split=self.split						,				verification_mode=lowerCAmelCase_						,				in_memory=self.keep_in_memory )
								return dataset




class 							snake_case							:

						'''simple docstring'''



						def __init__(							self       :      Dict						,				__lowercase       :      Tuple						,				__lowercase       :      List[Any]						,				__lowercase       :      List[Any] = None						,				__lowercase       :      str = None						,				**__lowercase       :      Dict						,				):


								'''simple docstring'''




								if num_proc is not None and num_proc <= 0:
										raise ValueError(f'''num_proc {num_proc} must be an integer > 0.''' )

								__UpperCAmelCase    :						Union[str, Any]       =						dataset
								__UpperCAmelCase    :						List[Any]       =						path_or_buf
								__UpperCAmelCase    :						Tuple       =						batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
								__UpperCAmelCase    :						str       =						num_proc
								__UpperCAmelCase    :						Optional[Any]       =						'''utf-8'''
								__UpperCAmelCase    :						List[str]       =						to_json_kwargs

						def       A_					(							self       :      Union[str, Any] ):


								'''simple docstring'''




								__UpperCAmelCase    :						Dict       =						self.to_json_kwargs.pop('''path_or_buf'''						,				lowerCAmelCase_ )
								__UpperCAmelCase    :						Dict       =						self.to_json_kwargs.pop('''orient'''						,				'''records''' )
								__UpperCAmelCase    :						Union[str, Any]       =						self.to_json_kwargs.pop('''lines'''						,				True if orient == '''records''' else False )
								__UpperCAmelCase    :						int       =						self.to_json_kwargs.pop('''index'''						,				False if orient in ['''split''', '''table'''] else True )
								__UpperCAmelCase    :						List[Any]       =						self.to_json_kwargs.pop('''compression'''						,				lowerCAmelCase_ )

								if compression not in [None, "infer", "gzip", "bz2", "xz"]:
										raise NotImplementedError(f'''`datasets` currently does not support {compression} compression''' )

								if isinstance(self.path_or_buf						,				(str, bytes, os.PathLike) ):
										with fsspec.open(self.path_or_buf						,				'''wb'''						,				compression=lowerCAmelCase_ ) as buffer:
												__UpperCAmelCase    :						List[str]       =						self._write(file_obj=lowerCAmelCase_						,				orient=lowerCAmelCase_						,				lines=lowerCAmelCase_						,				index=lowerCAmelCase_						,				**self.to_json_kwargs )
								else:
										if compression:
												raise NotImplementedError(
												    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
												    ''' was passed. Please provide a local path instead.''' )
										__UpperCAmelCase    :						int       =						self._write(
										    file_obj=self.path_or_buf						,				orient=lowerCAmelCase_						,				lines=lowerCAmelCase_						,				index=lowerCAmelCase_						,				**self.to_json_kwargs )
								return written

						def       A_					(							self       :      Any						,				__lowercase       :      Union[str, Any] ):


								'''simple docstring'''




								__UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						List[str]       =						args

								__UpperCAmelCase    :						int       =						query_table(
								    table=self.dataset.data						,				key=slice(lowerCAmelCase_						,				offset + self.batch_size )						,				indices=self.dataset._indices						,				)
								__UpperCAmelCase    :						List[str]       =						batch.to_pandas().to_json(
								    path_or_buf=lowerCAmelCase_						,				orient=lowerCAmelCase_						,				lines=lowerCAmelCase_						,				index=lowerCAmelCase_						,				**lowerCAmelCase_ )
								if not json_str.endswith('''\n''' ):
										json_str += "\n"
								return json_str.encode(self.encoding )







						def       A_					(							self       :      int						,				__lowercase       :      Dict						,				__lowercase       :      List[str]						,				__lowercase       :      Any						,				__lowercase       :      int						,				**__lowercase       :      Dict						,				):


								'''simple docstring'''




								__UpperCAmelCase    :						Optional[int]       =						0

								if self.num_proc is None or self.num_proc == 1:
										for offset in logging.tqdm(
										    range(0						,				len(self.dataset )						,				self.batch_size )						,				unit='''ba'''						,				disable=not logging.is_progress_bar_enabled()						,				desc='''Creating json from Arrow format'''						,				):
												__UpperCAmelCase    :						Union[str, Any]       =						self._batch_json((offset, orient, lines, index, to_json_kwargs) )
												written += file_obj.write(lowerCAmelCase_ )
								else:
										__UpperCAmelCase       ,     __UpperCAmelCase    :						List[Any]       =						len(self.dataset ), self.batch_size
										with multiprocessing.Pool(self.num_proc ) as pool:
												for json_str in logging.tqdm(
												    pool.imap(
												        self._batch_json						,				[(offset, orient, lines, index, to_json_kwargs) for offset in range(0						,				lowerCAmelCase_						,				lowerCAmelCase_ )]						,				)						,				total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,				unit='''ba'''						,				disable=not logging.is_progress_bar_enabled()						,				desc='''Creating json from Arrow format'''						,				):
														written += file_obj.write(lowerCAmelCase_ )

								return written






"""simple docstring"""





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      List[Any] ):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						Dict       =						nn.Linear(3						,				4 )
        __UpperCAmelCase    :						Union[str, Any]       =						nn.BatchNormad(4 )
        __UpperCAmelCase    :						List[str]       =						nn.Linear(4						,				5 )







      def       A_					(							self       :      Any						,				__lowercase       :      Any ):


        '''simple docstring'''




        return self.lineara(self.batchnorm(self.lineara(__lowercase ) ) )




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Optional[int]						,				*__lowercase       :      str						,				**__lowercase       :      Optional[int] ):


        '''simple docstring'''




        return (args[0] + 1,) + args[1:], kwargs




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def       A_					(							self       :      Any						,				__lowercase       :      Tuple						,				__lowercase       :      Any ):


        '''simple docstring'''




        return output + 1




class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()
        __UpperCAmelCase    :						Optional[int]       =						ModelHook()

        add_hook_to_module(__lowercase						,				__lowercase )
        self.assertEqual(test_model._hf_hook						,				__lowercase )
        self.assertTrue(hasattr(__lowercase						,				'''_old_forward''' ) )

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__						,				'''forward''' )
        self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )						,				['''x'''] )

        remove_hook_from_module(__lowercase )
        self.assertFalse(hasattr(__lowercase						,				'''_hf_hook''' ) )
        self.assertFalse(hasattr(__lowercase						,				'''_old_forward''' ) )

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						ModelForTest()
        __UpperCAmelCase    :						Tuple       =						ModelHook()

        add_hook_to_module(__lowercase						,				__lowercase )
        add_hook_to_module(__lowercase						,				__lowercase						,				append=__lowercase )

        self.assertEqual(isinstance(test_model._hf_hook						,				__lowercase )						,				__lowercase )
        self.assertEqual(len(test_model._hf_hook.hooks )						,				2 )
        self.assertTrue(hasattr(__lowercase						,				'''_old_forward''' ) )

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__						,				'''forward''' )
        self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )						,				['''x'''] )

        remove_hook_from_module(__lowercase )
        self.assertFalse(hasattr(__lowercase						,				'''_hf_hook''' ) )
        self.assertFalse(hasattr(__lowercase						,				'''_old_forward''' ) )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()
        __UpperCAmelCase    :						Tuple       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						test_model(x + 1 )
        __UpperCAmelCase    :						Optional[Any]       =						test_model(x + 2 )

        __UpperCAmelCase    :						Optional[int]       =						PreForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Union[str, Any]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 ) )

        # Attaching a hook to a model when it already has one replaces, does not chain
        __UpperCAmelCase    :						int       =						PreForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Any       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 ) )

        # You need to use the sequential hook to chain two or more hooks
        __UpperCAmelCase    :						Any       =						SequentialHook(PreForwardHook()						,				PreForwardHook() )
        add_hook_to_module(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						Tuple       =						test_model(__lowercase )
        assert torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						ModelForTest()
        __UpperCAmelCase    :						List[Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Tuple       =						test_model(__lowercase )

        __UpperCAmelCase    :						int       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1						,				atol=1e-5 ) )

        # Attaching a hook to a model when it already has one replaces, does not chain
        __UpperCAmelCase    :						str       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						List[str]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1						,				atol=1e-5 ) )

        # You need to use the sequential hook to chain two or more hooks
        __UpperCAmelCase    :						Optional[int]       =						SequentialHook(PostForwardHook()						,				PostForwardHook() )
        add_hook_to_module(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						Dict       =						test_model(__lowercase )
        assert torch.allclose(__lowercase						,				output + 2						,				atol=1e-5 )

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						ModelForTest()
        __UpperCAmelCase    :						Union[str, Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						str       =						test_model(__lowercase )

        __UpperCAmelCase    :						Union[str, Any]       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1 ) )
        self.assertTrue(outputa.requires_grad )

        __UpperCAmelCase    :						Optional[Any]       =						True
        __UpperCAmelCase    :						int       =						test_model(__lowercase )
        self.assertFalse(outputa.requires_grad )

      @require_multi_gpu
      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						ModelForTest()
        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        add_hook_to_module(model.lineara						,				AlignDevicesHook(execution_device=0 ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(execution_device=0 ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(execution_device=1 ) )

        self.assertEqual(model.lineara.weight.device						,				torch.device(0 ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device(0 ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device(0 ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device(1 ) )

        # We can still make a forward pass. The input does not need to be on any particular device
        __UpperCAmelCase    :						Dict       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Any       =						model(__lowercase )
        self.assertEqual(output.device						,				torch.device(1 ) )

        # We can add a general hook to put back output on same device as input.
        add_hook_to_module(__lowercase						,				AlignDevicesHook(io_same_device=__lowercase ) )
        __UpperCAmelCase    :						List[Any]       =						torch.randn(2						,				3 ).to(0 )
        __UpperCAmelCase    :						int       =						model(__lowercase )
        self.assertEqual(output.device						,				torch.device(0 ) )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						Tuple       =						{'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Optional[int]       =						torch.device(hook_kwargs['''execution_device'''] )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						int       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara )
        remove_hook_from_module(model.batchnorm )
        remove_hook_from_module(model.lineara )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        __UpperCAmelCase    :						str       =						{
            '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
            '''offload''': True,
            '''offload_buffers''': True,
        }

        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						Optional[Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						List[Any]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara )
        remove_hook_from_module(model.batchnorm )
        remove_hook_from_module(model.lineara )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						Optional[Any]       =						0 if torch.cuda.is_available() else '''cpu'''
        attach_align_device_hook(__lowercase						,				execution_device=__lowercase						,				offload=__lowercase )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Dict       =						torch.device(__lowercase )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Dict       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        attach_align_device_hook(__lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				offload_buffers=__lowercase )

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						Dict       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						str       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )







      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						str       =						0 if torch.cuda.is_available() else '''cpu'''
        attach_align_device_hook(
            __lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				weights_map=model.state_dict() )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Optional[Any]       =						torch.device(__lowercase )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						Any       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Dict       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        attach_align_device_hook(
            __lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				weights_map=model.state_dict()						,				offload_buffers=__lowercase						,				)

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						List[str]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )