

'''simple docstring'''





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
							from ...onnx.config import PatchingSpec
							from ...tokenization_utils_base import PreTrainedTokenizerBase


UpperCamelCase_    						=       logging.get_logger(__name__)

UpperCamelCase_    						=       {
    """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096""": """https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096-finetuned-triviaqa""": (
        """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"""
    ),
    """allenai/longformer-base-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
    """allenai/longformer-large-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
}





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):
				lowerCamelCase_		       =    'longformer'







				def __init__(       self     :		List[str]		,					UpperCAmelCase__     :		Union[List[int], int] = 512		,					UpperCAmelCase__     :		int = 2		,					UpperCAmelCase__     :		int = 1		,					UpperCAmelCase__     :		int = 0		,					UpperCAmelCase__     :		int = 2		,					UpperCAmelCase__     :		int = 30522		,					UpperCAmelCase__     :		int = 768		,					UpperCAmelCase__     :		int = 12		,					UpperCAmelCase__     :		int = 12		,					UpperCAmelCase__     :		int = 3072		,					UpperCAmelCase__     :		str = "gelu"		,					UpperCAmelCase__     :		float = 0.1		,					UpperCAmelCase__     :		float = 0.1		,					UpperCAmelCase__     :		int = 512		,					UpperCAmelCase__     :		int = 2		,					UpperCAmelCase__     :		float = 0.02		,					UpperCAmelCase__     :		float = 1E-12		,					UpperCAmelCase__     :		bool = False		,					**UpperCAmelCase__     :		Optional[int]		,					):
										'''simple docstring'''
										super().__init__(pad_token_id=UpperCAmelCase__		,					**UpperCAmelCase__   )

										lowercase :							Dict    		=attention_window
										lowercase :							Any    		=sep_token_id
										lowercase :							List[Any]    		=bos_token_id
										lowercase :							Optional[int]    		=eos_token_id
										lowercase :							List[str]    		=vocab_size
										lowercase :							Any    		=hidden_size
										lowercase :							Optional[int]    		=num_hidden_layers
										lowercase :							Optional[Any]    		=num_attention_heads
										lowercase :							List[Any]    		=hidden_act
										lowercase :							Optional[int]    		=intermediate_size
										lowercase :							str    		=hidden_dropout_prob
										lowercase :							str    		=attention_probs_dropout_prob
										lowercase :							Tuple    		=max_position_embeddings
										lowercase :							List[Any]    		=type_vocab_size
										lowercase :							Optional[Any]    		=initializer_range
										lowercase :							int    		=layer_norm_eps
										lowercase :							List[Any]    		=onnx_export





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):







				def __init__(       self     :		Any		,					UpperCAmelCase__     :		"PretrainedConfig"		,					UpperCAmelCase__     :		str = "default"		,					UpperCAmelCase__     :		"List[PatchingSpec]" = None   ):
										'''simple docstring'''
										super().__init__(UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__   )
										lowercase :							Tuple    		=True







				@property
				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										if self.task == "multiple-choice":
																lowercase :							Optional[Any]    		={0: '''batch''', 1: '''choice''', 2: '''sequence'''}
										else:
																lowercase :							Union[str, Any]    		={0: '''batch''', 1: '''sequence'''}
										return OrderedDict(
										    [
										        ('''input_ids''', dynamic_axis),
										        ('''attention_mask''', dynamic_axis),
										        ('''global_attention_mask''', dynamic_axis),
										    ]   )







				@property
				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							Any    		=super().outputs
										if self.task == "default":
																lowercase :							List[Any]    		={0: '''batch'''}
										return outputs







				@property
				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										return 1E-4







				@property
				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										# needs to be >= 14 to support tril operator
										return max(super().default_onnx_opset		,					14   )







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		"PreTrainedTokenizerBase"		,					UpperCAmelCase__     :		int = -1		,					UpperCAmelCase__     :		int = -1		,					UpperCAmelCase__     :		bool = False		,					UpperCAmelCase__     :		Optional[TensorType] = None		,					):
										'''simple docstring'''
										lowercase :							Optional[int]    		=super().generate_dummy_inputs(
										    preprocessor=UpperCAmelCase__		,					batch_size=UpperCAmelCase__		,					seq_length=UpperCAmelCase__		,					is_pair=UpperCAmelCase__		,					framework=UpperCAmelCase__   )
										import torch

										# for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
										# makes the export fail randomly
										lowercase :							Any    		=torch.zeros_like(inputs['''input_ids''']   )
										# make every second token global
										lowercase :							List[str]    		=1

										return inputs










'''simple docstring'''





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class __A     (    nn.Module		):



 '''simple docstring'''







 def __init__(self			)       ->	Dict:


      """simple docstring"""



      super().__init__()
      _a           =    nn.Linear(3					,  4			)
      _a           =    nn.BatchNormad(4			)
      _a           =    nn.Linear(4					,  5			)



 def    a__    (self					,  A			)       ->	Dict:


      """simple docstring"""



      return self.lineara(self.batchnorm(self.lineara(A			)			)			)






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  *A					,  **A			)       ->	Optional[Any]:


      """simple docstring"""



      return (args[0] + 1,) + args[1:], kwargs






class __A     (    A		):



 '''simple docstring'''







 def    a__    (self					,  A					,  A			)       ->	int:


      """simple docstring"""



      return output + 1






class __A     (    unittest.TestCase		):



 '''simple docstring'''







 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      self.assertEqual(test_model._hf_hook					,  A			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    ModelHook()

      add_hook_to_module(A					,  A			)
      add_hook_to_module(A					,  A					,  append=A			)

      self.assertEqual(isinstance(test_model._hf_hook					,  A			)					,  A			)
      self.assertEqual(len(test_model._hf_hook.hooks			)					,  2			)
      self.assertTrue(hasattr(A					,  '''_old_forward'''			)			)

      # Check adding the hook did not change the name or the signature
      self.assertEqual(test_model.forward.__name__					,  '''forward'''			)
      self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)					,  ['''x''']			)

      remove_hook_from_module(A			)
      self.assertFalse(hasattr(A					,  '''_hf_hook'''			)			)
      self.assertFalse(hasattr(A					,  '''_old_forward'''			)			)



 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(x + 1			)
      _a           =    test_model(x + 2			)

      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PreForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PreForwardHook()					,  PreForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  A					,  atol=1E-5			)



 def    a__    (self			)       ->	str:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # Attaching a hook to a model when it already has one replaces, does not chain
      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1					,  atol=1E-5			)			)

      # You need to use the sequential hook to chain two or more hooks
      _a           =    SequentialHook(PostForwardHook()					,  PostForwardHook()			)
      add_hook_to_module(A					,  A			)

      _a           =    test_model(A			)
      assert torch.allclose(A					,  output + 2					,  atol=1E-5			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()
      _a           =    torch.randn(2					,  3			)
      _a           =    test_model(A			)

      _a           =    PostForwardHook()
      add_hook_to_module(A					,  A			)
      _a           =    test_model(A			)
      self.assertTrue(torch.allclose(A					,  output + 1			)			)
      self.assertTrue(outputa.requires_grad			)

      _a           =    True
      _a           =    test_model(A			)
      self.assertFalse(outputa.requires_grad			)



 @require_multi_gpu
 def    a__    (self			)       ->	List[Any]:


      """simple docstring"""



      _a           =    ModelForTest()
      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(execution_device=0			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(execution_device=1			)			)

      self.assertEqual(model.lineara.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device(0			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device(0			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device(1			)			)

      # We can still make a forward pass. The input does not need to be on any particular device
      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(1			)			)

      # We can add a general hook to put back output on same device as input.
      add_hook_to_module(A					,  AlignDevicesHook(io_same_device=A			)			)
      _a           =    torch.randn(2					,  3			).to(0			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  torch.device(0			)			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    {'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(hook_kwargs['''execution_device''']			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      _a           =    {
          '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
          '''offload''': True,
          '''offload_buffers''': True,
      }

      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.batchnorm					,  AlignDevicesHook(**A			)			)
      add_hook_to_module(model.lineara					,  AlignDevicesHook(**A			)			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_module(model.lineara			)
      remove_hook_from_module(model.batchnorm			)
      remove_hook_from_module(model.lineara			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(A					,  execution_device=A					,  offload=A			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(A					,  execution_device=A					,  offload=A					,  offload_buffers=A			)

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a           =    ModelForTest()

      # Everything is on CPU
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # This will move each submodule on different devices
      _a           =    0 if torch.cuda.is_available() else '''cpu'''
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()			)

      # Parameters have been offloaded, so on the meta device
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      # Buffers are not included in the offload by default, so are on the execution device
      _a           =    torch.device(A			)
      self.assertEqual(model.batchnorm.running_mean.device					,  A			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)

      # Now test with buffers included in the offload
      attach_align_device_hook(
          A					,  execution_device=A					,  offload=A					,  weights_map=model.state_dict()					,  offload_buffers=A					,  )

      # Parameters have been offloaded, so on the meta device, buffers included
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''meta'''			)			)
      self.assertEqual(model.batchnorm.running_mean.device					,  torch.device('''meta'''			)			)

      _a           =    torch.randn(2					,  3			)
      _a           =    model(A			)
      self.assertEqual(output.device					,  A			)

      # Removing hooks loads back the weights in the model.
      remove_hook_from_submodules(A			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.batchnorm.weight.device					,  torch.device('''cpu'''			)			)
      self.assertEqual(model.lineara.weight.device					,  torch.device('''cpu'''			)			)



