





"""simple docstring"""





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast




class 							snake_case							(			unittest.TestCase ):

						'''simple docstring'''



						def       A_					(							self       :      Any ):


								'''simple docstring'''




								__UpperCAmelCase    :						Tuple       =						0

						@slow
						def       A_					(							self       :      Dict ):


								'''simple docstring'''




								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
										__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained(__lowercase )
										self.assertIsNotNone(__lowercase )
										self.assertIsInstance(__lowercase						,				(BertTokenizer, BertTokenizerFast) )
										self.assertGreater(len(__lowercase )						,				0 )

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
										__UpperCAmelCase    :						List[Any]       =						AutoTokenizer.from_pretrained(__lowercase )
										self.assertIsNotNone(__lowercase )
										self.assertIsInstance(__lowercase						,				(GPTaTokenizer, GPTaTokenizerFast) )
										self.assertGreater(len(__lowercase )						,				0 )

						def       A_					(							self       :      Any ):


								'''simple docstring'''




								__UpperCAmelCase    :						Tuple       =						AutoTokenizer.from_pretrained(__lowercase )
								self.assertIsInstance(__lowercase						,				(BertTokenizer, BertTokenizerFast) )
								self.assertEqual(tokenizer.vocab_size						,				12 )

						def       A_					(							self       :      Any ):


								'''simple docstring'''




								__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained(__lowercase )
								self.assertIsInstance(__lowercase						,				(RobertaTokenizer, RobertaTokenizerFast) )
								self.assertEqual(tokenizer.vocab_size						,				20 )

						def       A_					(							self       :      Dict ):


								'''simple docstring'''




								__UpperCAmelCase    :						Optional[int]       =						AutoConfig.from_pretrained(__lowercase )
								self.assertIsInstance(__lowercase						,				__lowercase )
								# Check that tokenizer_type ≠ model_type
								__UpperCAmelCase    :						Tuple       =						AutoTokenizer.from_pretrained(__lowercase						,				config=__lowercase )
								self.assertIsInstance(__lowercase						,				(BertTokenizer, BertTokenizerFast) )
								self.assertEqual(tokenizer.vocab_size						,				12 )

						def       A_					(							self       :      List[str] ):


								'''simple docstring'''




								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.txt'''						,				os.path.join(__lowercase						,				'''vocab.txt''' ) )

										__UpperCAmelCase    :						Dict       =						AutoTokenizer.from_pretrained(__lowercase						,				tokenizer_type='''bert'''						,				use_fast=__lowercase )
										self.assertIsInstance(__lowercase						,				__lowercase )

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.json'''						,				os.path.join(__lowercase						,				'''vocab.json''' ) )
										shutil.copy('''./tests/fixtures/merges.txt'''						,				os.path.join(__lowercase						,				'''merges.txt''' ) )

										__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained(__lowercase						,				tokenizer_type='''gpt2'''						,				use_fast=__lowercase )
										self.assertIsInstance(__lowercase						,				__lowercase )

						@require_tokenizers
						def       A_					(							self       :      str ):


								'''simple docstring'''




								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.txt'''						,				os.path.join(__lowercase						,				'''vocab.txt''' ) )

										__UpperCAmelCase    :						Dict       =						AutoTokenizer.from_pretrained(__lowercase						,				tokenizer_type='''bert''' )
										self.assertIsInstance(__lowercase						,				__lowercase )

								with tempfile.TemporaryDirectory() as tmp_dir:
										shutil.copy('''./tests/fixtures/vocab.json'''						,				os.path.join(__lowercase						,				'''vocab.json''' ) )
										shutil.copy('''./tests/fixtures/merges.txt'''						,				os.path.join(__lowercase						,				'''merges.txt''' ) )

										__UpperCAmelCase    :						Optional[int]       =						AutoTokenizer.from_pretrained(__lowercase						,				tokenizer_type='''gpt2''' )
										self.assertIsInstance(__lowercase						,				__lowercase )

						def       A_					(							self       :      Tuple ):


								'''simple docstring'''




								with pytest.raises(__lowercase ):
										AutoTokenizer.from_pretrained('''./'''						,				tokenizer_type='''xxx''' )

						@require_tokenizers
						def       A_					(							self       :      Dict ):


								'''simple docstring'''




								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										__UpperCAmelCase    :						List[Any]       =						tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
										self.assertIsInstance(__lowercase						,				(BertTokenizer, BertTokenizerFast) )

										if isinstance(__lowercase						,				__lowercase ):
												self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,				__lowercase )
										else:
												self.assertEqual(tokenizer.do_lower_case						,				__lowercase )

										self.assertEqual(tokenizer.model_max_length						,				512 )

						@require_tokenizers
						def       A_					(							self       :      Any ):


								'''simple docstring'''




								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
										with self.assertRaisesRegex(
										    __lowercase						,				'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''						,				):
												__UpperCAmelCase    :						List[str]       =						tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )

						def       A_					(							self       :      Any ):


								'''simple docstring'''




								__UpperCAmelCase    :						Dict       =						TOKENIZER_MAPPING.values()
								__UpperCAmelCase    :						str       =						[]

								for slow_tok, fast_tok in tokenizers:
										if slow_tok is not None:
												tokenizer_names.append(slow_tok.__name__ )

										if fast_tok is not None:
												tokenizer_names.append(fast_tok.__name__ )

								for tokenizer_name in tokenizer_names:
										# must find the right class
										tokenizer_class_from_name(__lowercase )

						@require_tokenizers
						def       A_					(							self       :      Optional[int] ):


								'''simple docstring'''




								self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''						,				use_fast=__lowercase )						,				__lowercase )
								self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' )						,				__lowercase )

						@require_tokenizers
						def       A_					(							self       :      Optional[int] ):


								'''simple docstring'''




								__UpperCAmelCase    :						List[Any]       =						AutoTokenizer.from_pretrained('''distilbert-base-uncased'''						,				do_lower_case=__lowercase )
								__UpperCAmelCase    :						Optional[Any]       =						'''Hello, world. How are you?'''
								__UpperCAmelCase    :						Optional[int]       =						tokenizer.tokenize(__lowercase )
								self.assertEqual('''[UNK]'''						,				tokens[0] )

								__UpperCAmelCase    :						List[Any]       =						AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''						,				do_lower_case=__lowercase )
								__UpperCAmelCase    :						Optional[int]       =						tokenizer.tokenize(__lowercase )
								self.assertEqual('''[UNK]'''						,				tokens[0] )

						@require_tokenizers
						def       A_					(							self       :      Tuple ):


								'''simple docstring'''




								__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
								self.assertEqual(type(__lowercase )						,				__lowercase )
								self.assertEqual(tokenizer.model_max_length						,				512 )
								self.assertEqual(tokenizer.vocab_size						,				30_000 )
								self.assertEqual(tokenizer.unk_token						,				'''[UNK]''' )
								self.assertEqual(tokenizer.padding_side						,				'''right''' )
								self.assertEqual(tokenizer.truncation_side						,				'''right''' )

						def       A_					(							self       :      Dict ):


								'''simple docstring'''




								__UpperCAmelCase    :						Optional[int]       =						AutoTokenizer.from_pretrained(__lowercase )
								self.assertIsInstance(__lowercase						,				(BertTokenizer, BertTokenizerFast) )
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__lowercase )
										__UpperCAmelCase    :						List[Any]       =						AutoTokenizer.from_pretrained(__lowercase )

								self.assertIsInstance(__lowercase						,				tokenizer.__class__ )
								self.assertEqual(tokenizera.vocab_size						,				12 )

						def       A_					(							self       :      Any ):


								'''simple docstring'''




								__UpperCAmelCase    :						str       =						AutoTokenizer.from_pretrained('''ctrl''' )
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(__lowercase						,				__lowercase )

						def       A_					(							self       :      int ):


								'''simple docstring'''




								__UpperCAmelCase    :						int       =						get_tokenizer_config('''bert-base-cased''' )
								__UpperCAmelCase    :						List[str]       =						config.pop('''_commit_hash'''						,				__lowercase )
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(__lowercase						,				{'''do_lower_case''': False} )

								# This model does not have a tokenizer_config so we get back an empty dict.
								__UpperCAmelCase    :						Union[str, Any]       =						get_tokenizer_config(__lowercase )
								self.assertDictEqual(__lowercase						,				{} )

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								__UpperCAmelCase    :						Any       =						AutoTokenizer.from_pretrained(__lowercase )
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__lowercase )
										__UpperCAmelCase    :						List[str]       =						get_tokenizer_config(__lowercase )

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config['''tokenizer_class''']						,				'''BertTokenizer''' )

						def       A_					(							self       :      List[str] ):


								'''simple docstring'''




								try:
										AutoConfig.register('''custom'''						,				__lowercase )

										AutoTokenizer.register(__lowercase						,				slow_tokenizer_class=__lowercase )
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(__lowercase ):
												AutoTokenizer.register(__lowercase						,				slow_tokenizer_class=__lowercase )

										__UpperCAmelCase    :						Tuple       =						CustomTokenizer.from_pretrained(__lowercase )
										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(__lowercase )

												__UpperCAmelCase    :						Any       =						AutoTokenizer.from_pretrained(__lowercase )
												self.assertIsInstance(__lowercase						,				__lowercase )

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						@require_tokenizers
						def       A_					(							self       :      Tuple ):


								'''simple docstring'''




								try:
										AutoConfig.register('''custom'''						,				__lowercase )

										# Can register in two steps
										AutoTokenizer.register(__lowercase						,				slow_tokenizer_class=__lowercase )
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, None) )
										AutoTokenizer.register(__lowercase						,				fast_tokenizer_class=__lowercase )
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, CustomTokenizerFast) )

										del TOKENIZER_MAPPING._extra_content[CustomConfig]
										# Can register in one step
										AutoTokenizer.register(
										    __lowercase						,				slow_tokenizer_class=__lowercase						,				fast_tokenizer_class=__lowercase )
										self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, CustomTokenizerFast) )

										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(__lowercase ):
												AutoTokenizer.register(__lowercase						,				fast_tokenizer_class=__lowercase )

										# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
										# and that model does not have a tokenizer.json
										with tempfile.TemporaryDirectory() as tmp_dir:
												__UpperCAmelCase    :						Any       =						BertTokenizerFast.from_pretrained(__lowercase )
												bert_tokenizer.save_pretrained(__lowercase )
												__UpperCAmelCase    :						Optional[Any]       =						CustomTokenizerFast.from_pretrained(__lowercase )

										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(__lowercase )

												__UpperCAmelCase    :						Tuple       =						AutoTokenizer.from_pretrained(__lowercase )
												self.assertIsInstance(__lowercase						,				__lowercase )

												__UpperCAmelCase    :						Tuple       =						AutoTokenizer.from_pretrained(__lowercase						,				use_fast=__lowercase )
												self.assertIsInstance(__lowercase						,				__lowercase )

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						def       A_					(							self       :      Dict ):


								'''simple docstring'''




								with self.assertRaises(__lowercase ):
										__UpperCAmelCase    :						Union[str, Any]       =						AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__lowercase ):
										__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase )

								__UpperCAmelCase    :						Dict       =						AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase )
								self.assertTrue(tokenizer.special_attribute_present )
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__lowercase )
										__UpperCAmelCase    :						Any       =						AutoTokenizer.from_pretrained(__lowercase						,				trust_remote_code=__lowercase )
								self.assertTrue(reloaded_tokenizer.special_attribute_present )

								if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )

										# Test we can also load the slow version
										__UpperCAmelCase    :						Any       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase						,				use_fast=__lowercase )
										self.assertTrue(tokenizer.special_attribute_present )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(__lowercase )
												__UpperCAmelCase    :						List[str]       =						AutoTokenizer.from_pretrained(__lowercase						,				trust_remote_code=__lowercase						,				use_fast=__lowercase )
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										self.assertTrue(reloaded_tokenizer.special_attribute_present )
								else:
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										self.assertEqual(reloaded_tokenizer.__class__.__name__						,				'''NewTokenizer''' )

						@require_tokenizers
						def       A_					(							self       :      Optional[Any] ):


								'''simple docstring'''




								class 							snake_case							(			__UpperCAmelCase ):

														'''simple docstring'''



														_A     :       Optional[Any]         =       False




								class 							snake_case							(			__UpperCAmelCase ):

														'''simple docstring'''



														_A     :       Tuple         =       NewTokenizer
														_A     :       Dict         =       False

								try:
										AutoConfig.register('''custom'''						,				__lowercase )
										AutoTokenizer.register(__lowercase						,				slow_tokenizer_class=__lowercase )
										AutoTokenizer.register(__lowercase						,				fast_tokenizer_class=__lowercase )
										# If remote code is not set, the default is to use local
										__UpperCAmelCase    :						int       =						AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )
										self.assertFalse(tokenizer.special_attribute_present )
										__UpperCAmelCase    :						int       =						AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''						,				use_fast=__lowercase )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										self.assertFalse(tokenizer.special_attribute_present )

										# If remote code is disabled, we load the local one.
										__UpperCAmelCase    :						str       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )
										self.assertFalse(tokenizer.special_attribute_present )
										__UpperCAmelCase    :						Optional[Any]       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase						,				use_fast=__lowercase )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										self.assertFalse(tokenizer.special_attribute_present )

										# If remote is enabled, we load from the Hub
										__UpperCAmelCase    :						int       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )
										self.assertTrue(tokenizer.special_attribute_present )
										__UpperCAmelCase    :						Optional[int]       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer'''						,				trust_remote_code=__lowercase						,				use_fast=__lowercase )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
										self.assertTrue(tokenizer.special_attribute_present )

								finally:
										if "custom" in CONFIG_MAPPING._extra_content:
												del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
												del TOKENIZER_MAPPING._extra_content[CustomConfig]

						def       A_					(							self       :      Union[str, Any] ):


								'''simple docstring'''




								__UpperCAmelCase    :						Any       =						AutoTokenizer.from_pretrained(
								    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''						,				trust_remote_code=__lowercase )
								self.assertTrue(tokenizer.special_attribute_present )
								if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizerFast''' )

										# Test we can also load the slow version
										__UpperCAmelCase    :						Optional[int]       =						AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''						,				trust_remote_code=__lowercase						,				use_fast=__lowercase )
										self.assertTrue(tokenizer.special_attribute_present )
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )
								else:
										self.assertEqual(tokenizer.__class__.__name__						,				'''NewTokenizer''' )

						def       A_					(							self       :      Any ):


								'''simple docstring'''




								with self.assertRaisesRegex(
								    __lowercase						,				'''bert-base is not a local folder and is not a valid model identifier''' ):
										__UpperCAmelCase    :						Optional[Any]       =						AutoTokenizer.from_pretrained('''bert-base''' )

						def       A_					(							self       :      List[Any] ):


								'''simple docstring'''




								with self.assertRaisesRegex(
								    __lowercase						,				R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
										__UpperCAmelCase    :						Optional[Any]       =						AutoTokenizer.from_pretrained(__lowercase						,				revision='''aaaaaa''' )







						def       A_					(							self       :      str ):


								'''simple docstring'''




								__UpperCAmelCase    :						str       =						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
								with RequestCounter() as counter:
										__UpperCAmelCase    :						Tuple       =						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
										self.assertEqual(counter.get_request_count						,				0 )
										self.assertEqual(counter.head_request_count						,				1 )
										self.assertEqual(counter.other_request_count						,				0 )






"""simple docstring"""





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu




class 							snake_case							(			nn.Module ):

      '''simple docstring'''



      def __init__(							self       :      List[Any] ):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						Dict       =						nn.Linear(3						,				4 )
        __UpperCAmelCase    :						Union[str, Any]       =						nn.BatchNormad(4 )
        __UpperCAmelCase    :						List[str]       =						nn.Linear(4						,				5 )







      def       A_					(							self       :      Any						,				__lowercase       :      Any ):


        '''simple docstring'''




        return self.lineara(self.batchnorm(self.lineara(__lowercase ) ) )




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Optional[int]						,				*__lowercase       :      str						,				**__lowercase       :      Optional[int] ):


        '''simple docstring'''




        return (args[0] + 1,) + args[1:], kwargs




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def       A_					(							self       :      Any						,				__lowercase       :      Tuple						,				__lowercase       :      Any ):


        '''simple docstring'''




        return output + 1




class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()
        __UpperCAmelCase    :						Optional[int]       =						ModelHook()

        add_hook_to_module(__lowercase						,				__lowercase )
        self.assertEqual(test_model._hf_hook						,				__lowercase )
        self.assertTrue(hasattr(__lowercase						,				'''_old_forward''' ) )

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__						,				'''forward''' )
        self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )						,				['''x'''] )

        remove_hook_from_module(__lowercase )
        self.assertFalse(hasattr(__lowercase						,				'''_hf_hook''' ) )
        self.assertFalse(hasattr(__lowercase						,				'''_old_forward''' ) )

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						ModelForTest()
        __UpperCAmelCase    :						Tuple       =						ModelHook()

        add_hook_to_module(__lowercase						,				__lowercase )
        add_hook_to_module(__lowercase						,				__lowercase						,				append=__lowercase )

        self.assertEqual(isinstance(test_model._hf_hook						,				__lowercase )						,				__lowercase )
        self.assertEqual(len(test_model._hf_hook.hooks )						,				2 )
        self.assertTrue(hasattr(__lowercase						,				'''_old_forward''' ) )

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__						,				'''forward''' )
        self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )						,				['''x'''] )

        remove_hook_from_module(__lowercase )
        self.assertFalse(hasattr(__lowercase						,				'''_hf_hook''' ) )
        self.assertFalse(hasattr(__lowercase						,				'''_old_forward''' ) )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()
        __UpperCAmelCase    :						Tuple       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						test_model(x + 1 )
        __UpperCAmelCase    :						Optional[Any]       =						test_model(x + 2 )

        __UpperCAmelCase    :						Optional[int]       =						PreForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Union[str, Any]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 ) )

        # Attaching a hook to a model when it already has one replaces, does not chain
        __UpperCAmelCase    :						int       =						PreForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Any       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 ) )

        # You need to use the sequential hook to chain two or more hooks
        __UpperCAmelCase    :						Any       =						SequentialHook(PreForwardHook()						,				PreForwardHook() )
        add_hook_to_module(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						Tuple       =						test_model(__lowercase )
        assert torch.allclose(__lowercase						,				__lowercase						,				atol=1e-5 )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						ModelForTest()
        __UpperCAmelCase    :						List[Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Tuple       =						test_model(__lowercase )

        __UpperCAmelCase    :						int       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1						,				atol=1e-5 ) )

        # Attaching a hook to a model when it already has one replaces, does not chain
        __UpperCAmelCase    :						str       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						List[str]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1						,				atol=1e-5 ) )

        # You need to use the sequential hook to chain two or more hooks
        __UpperCAmelCase    :						Optional[int]       =						SequentialHook(PostForwardHook()						,				PostForwardHook() )
        add_hook_to_module(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						Dict       =						test_model(__lowercase )
        assert torch.allclose(__lowercase						,				output + 2						,				atol=1e-5 )

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						ModelForTest()
        __UpperCAmelCase    :						Union[str, Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						str       =						test_model(__lowercase )

        __UpperCAmelCase    :						Union[str, Any]       =						PostForwardHook()
        add_hook_to_module(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						test_model(__lowercase )
        self.assertTrue(torch.allclose(__lowercase						,				output + 1 ) )
        self.assertTrue(outputa.requires_grad )

        __UpperCAmelCase    :						Optional[Any]       =						True
        __UpperCAmelCase    :						int       =						test_model(__lowercase )
        self.assertFalse(outputa.requires_grad )

      @require_multi_gpu
      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						ModelForTest()
        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        add_hook_to_module(model.lineara						,				AlignDevicesHook(execution_device=0 ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(execution_device=0 ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(execution_device=1 ) )

        self.assertEqual(model.lineara.weight.device						,				torch.device(0 ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device(0 ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device(0 ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device(1 ) )

        # We can still make a forward pass. The input does not need to be on any particular device
        __UpperCAmelCase    :						Dict       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Any       =						model(__lowercase )
        self.assertEqual(output.device						,				torch.device(1 ) )

        # We can add a general hook to put back output on same device as input.
        add_hook_to_module(__lowercase						,				AlignDevicesHook(io_same_device=__lowercase ) )
        __UpperCAmelCase    :						List[Any]       =						torch.randn(2						,				3 ).to(0 )
        __UpperCAmelCase    :						int       =						model(__lowercase )
        self.assertEqual(output.device						,				torch.device(0 ) )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						Tuple       =						{'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Optional[int]       =						torch.device(hook_kwargs['''execution_device'''] )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						int       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara )
        remove_hook_from_module(model.batchnorm )
        remove_hook_from_module(model.lineara )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        __UpperCAmelCase    :						str       =						{
            '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
            '''offload''': True,
            '''offload_buffers''': True,
        }

        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.batchnorm						,				AlignDevicesHook(**__lowercase ) )
        add_hook_to_module(model.lineara						,				AlignDevicesHook(**__lowercase ) )

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						Optional[Any]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						List[Any]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara )
        remove_hook_from_module(model.batchnorm )
        remove_hook_from_module(model.lineara )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						Optional[Any]       =						0 if torch.cuda.is_available() else '''cpu'''
        attach_align_device_hook(__lowercase						,				execution_device=__lowercase						,				offload=__lowercase )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Dict       =						torch.device(__lowercase )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Dict       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        attach_align_device_hook(__lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				offload_buffers=__lowercase )

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						Dict       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						str       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )







      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # This will move each submodule on different devices
        __UpperCAmelCase    :						str       =						0 if torch.cuda.is_available() else '''cpu'''
        attach_align_device_hook(
            __lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				weights_map=model.state_dict() )

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        # Buffers are not included in the offload by default, so are on the execution device
        __UpperCAmelCase    :						Optional[Any]       =						torch.device(__lowercase )
        self.assertEqual(model.batchnorm.running_mean.device						,				__lowercase )

        __UpperCAmelCase    :						Any       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Dict       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )

        # Now test with buffers included in the offload
        attach_align_device_hook(
            __lowercase						,				execution_device=__lowercase						,				offload=__lowercase						,				weights_map=model.state_dict()						,				offload_buffers=__lowercase						,				)

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''meta''' ) )
        self.assertEqual(model.batchnorm.running_mean.device						,				torch.device('''meta''' ) )

        __UpperCAmelCase    :						List[str]       =						torch.randn(2						,				3 )
        __UpperCAmelCase    :						Optional[int]       =						model(__lowercase )
        self.assertEqual(output.device						,				__lowercase )

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(__lowercase )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.batchnorm.weight.device						,				torch.device('''cpu''' ) )
        self.assertEqual(model.lineara.weight.device						,				torch.device('''cpu''' ) )