




'''simple docstring'''






import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class        lowercase_      (						nn.Module							):





     def __init__(     self     :					Optional[int]			):
           super().__init__()
           snake_case__       :   int	      =    nn.Linear(3     ,							4			)
           snake_case__       :   int	      =    nn.BatchNormad(4			)
           snake_case__       :   Optional[Any]	      =    nn.Linear(4     ,							5			)





     def 			_lowerCAmelCase      (     self     :					Dict     ,							__lowerCamelCase     :					Tuple			):
           return self.lineara(self.batchnorm(self.lineara(__lowerCamelCase			)			)			)






class        lowercase_      (						lowerCAmelCase_							):





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]     ,							__lowerCamelCase     :					str     ,							*__lowerCamelCase     :					Any     ,							**__lowerCamelCase     :					Optional[Any]			):
           return (args[0] + 1,) + args[1:], kwargs






class        lowercase_      (						lowerCAmelCase_							):





     def 			_lowerCAmelCase      (     self     :					List[str]     ,							__lowerCamelCase     :					Dict     ,							__lowerCamelCase     :					str			):
           return output + 1






class        lowercase_      (						unittest.TestCase							):





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]			):
           snake_case__       :   Optional[Any]	      =    ModelForTest()
           snake_case__       :   str	      =    ModelHook()

           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           self.assertEqual(test_model._hf_hook     ,							__lowerCamelCase			)
           self.assertTrue(hasattr(__lowerCamelCase     ,							'_old_forward'			)			)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,							'forward'			)
           self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)     ,							['x']			)

           remove_hook_from_module(__lowerCamelCase			)
           self.assertFalse(hasattr(__lowerCamelCase     ,							'_hf_hook'			)			)
           self.assertFalse(hasattr(__lowerCamelCase     ,							'_old_forward'			)			)





     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   Optional[int]	      =    ModelForTest()
           snake_case__       :   Optional[int]	      =    ModelHook()

           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase     ,							append=__lowerCamelCase			)

           self.assertEqual(isinstance(test_model._hf_hook     ,							__lowerCamelCase			)     ,							__lowerCamelCase			)
           self.assertEqual(len(test_model._hf_hook.hooks			)     ,							2			)
           self.assertTrue(hasattr(__lowerCamelCase     ,							'_old_forward'			)			)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,							'forward'			)
           self.assertListEqual(list(inspect.signature(test_model.forward			).parameters			)     ,							['x']			)

           remove_hook_from_module(__lowerCamelCase			)
           self.assertFalse(hasattr(__lowerCamelCase     ,							'_hf_hook'			)			)
           self.assertFalse(hasattr(__lowerCamelCase     ,							'_old_forward'			)			)





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]			):
           snake_case__       :   str	      =    ModelForTest()
           snake_case__       :   List[Any]	      =    torch.randn(2     ,							3			)
           snake_case__       :   Union[str, Any]	      =    test_model(x + 1			)
           snake_case__       :   str	      =    test_model(x + 2			)

           snake_case__       :   int	      =    PreForwardHook()
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   Tuple	      =    test_model(__lowerCamelCase			)
           self.assertTrue(torch.allclose(__lowerCamelCase     ,							__lowerCamelCase     ,							atol=1E-5			)			)

           # Attaching a hook to a model when it already has one replaces, does not chain
           snake_case__       :   Optional[Any]	      =    PreForwardHook()
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   Dict	      =    test_model(__lowerCamelCase			)
           self.assertTrue(torch.allclose(__lowerCamelCase     ,							__lowerCamelCase     ,							atol=1E-5			)			)

           # You need to use the sequential hook to chain two or more hooks
           snake_case__       :   str	      =    SequentialHook(PreForwardHook()     ,							PreForwardHook()			)
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)

           snake_case__       :   Any	      =    test_model(__lowerCamelCase			)
           assert torch.allclose(__lowerCamelCase     ,							__lowerCamelCase     ,							atol=1E-5			)





     def 			_lowerCAmelCase      (     self     :					Dict			):
           snake_case__       :   Tuple	      =    ModelForTest()
           snake_case__       :   Dict	      =    torch.randn(2     ,							3			)
           snake_case__       :   Optional[Any]	      =    test_model(__lowerCamelCase			)

           snake_case__       :   Dict	      =    PostForwardHook()
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   str	      =    test_model(__lowerCamelCase			)
           self.assertTrue(torch.allclose(__lowerCamelCase     ,							output + 1     ,							atol=1E-5			)			)

           # Attaching a hook to a model when it already has one replaces, does not chain
           snake_case__       :   Union[str, Any]	      =    PostForwardHook()
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   List[Any]	      =    test_model(__lowerCamelCase			)
           self.assertTrue(torch.allclose(__lowerCamelCase     ,							output + 1     ,							atol=1E-5			)			)

           # You need to use the sequential hook to chain two or more hooks
           snake_case__       :   int	      =    SequentialHook(PostForwardHook()     ,							PostForwardHook()			)
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)

           snake_case__       :   List[str]	      =    test_model(__lowerCamelCase			)
           assert torch.allclose(__lowerCamelCase     ,							output + 2     ,							atol=1E-5			)





     def 			_lowerCAmelCase      (     self     :					Tuple			):
           snake_case__       :   List[Any]	      =    ModelForTest()
           snake_case__       :   str	      =    torch.randn(2     ,							3			)
           snake_case__       :   str	      =    test_model(__lowerCamelCase			)

           snake_case__       :   Union[str, Any]	      =    PostForwardHook()
           add_hook_to_module(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   Optional[int]	      =    test_model(__lowerCamelCase			)
           self.assertTrue(torch.allclose(__lowerCamelCase     ,							output + 1			)			)
           self.assertTrue(outputa.requires_grad			)

           snake_case__       :   Tuple	      =    True
           snake_case__       :   List[Any]	      =    test_model(__lowerCamelCase			)
           self.assertFalse(outputa.requires_grad			)





     @require_multi_gpu
     def 			_lowerCAmelCase      (     self     :					Tuple			):
           snake_case__       :   Any	      =    ModelForTest()
           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # This will move each submodule on different devices
           add_hook_to_module(model.lineara     ,							AlignDevicesHook(execution_device=0			)			)
           add_hook_to_module(model.batchnorm     ,							AlignDevicesHook(execution_device=0			)			)
           add_hook_to_module(model.lineara     ,							AlignDevicesHook(execution_device=1			)			)

           self.assertEqual(model.lineara.weight.device     ,							torch.device(0			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device(0			)			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							torch.device(0			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device(1			)			)

           # We can still make a forward pass. The input does not need to be on any particular device
           snake_case__       :   Optional[Any]	      =    torch.randn(2     ,							3			)
           snake_case__       :   str	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							torch.device(1			)			)

           # We can add a general hook to put back output on same device as input.
           add_hook_to_module(__lowerCamelCase     ,							AlignDevicesHook(io_same_device=__lowerCamelCase			)			)
           snake_case__       :   str	      =    torch.randn(2     ,							3			).to(0			)
           snake_case__       :   List[Any]	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							torch.device(0			)			)





     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   int	      =    ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # This will move each submodule on different devices
           snake_case__       :   Optional[int]	      =    {'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

           add_hook_to_module(model.lineara     ,							AlignDevicesHook(**__lowerCamelCase			)			)
           add_hook_to_module(model.batchnorm     ,							AlignDevicesHook(**__lowerCamelCase			)			)
           add_hook_to_module(model.lineara     ,							AlignDevicesHook(**__lowerCamelCase			)			)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           # Buffers are not included in the offload by default, so are on the execution device
           snake_case__       :   Dict	      =    torch.device(hook_kwargs['execution_device']			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							__lowerCamelCase			)

           snake_case__       :   Dict	      =    torch.randn(2     ,							3			)
           snake_case__       :   Dict	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara			)
           remove_hook_from_module(model.batchnorm			)
           remove_hook_from_module(model.lineara			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # Now test with buffers included in the offload
           snake_case__       :   Any	      =    {
               'execution_device': 0 if torch.cuda.is_available() else 'cpu',
               'offload': True,
               'offload_buffers': True,
           }

           add_hook_to_module(model.lineara     ,							AlignDevicesHook(**__lowerCamelCase			)			)
           add_hook_to_module(model.batchnorm     ,							AlignDevicesHook(**__lowerCamelCase			)			)
           add_hook_to_module(model.lineara     ,							AlignDevicesHook(**__lowerCamelCase			)			)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							torch.device('meta'			)			)

           snake_case__       :   List[Any]	      =    torch.randn(2     ,							3			)
           snake_case__       :   str	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara			)
           remove_hook_from_module(model.batchnorm			)
           remove_hook_from_module(model.lineara			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)





     def 			_lowerCAmelCase      (     self     :					Optional[Any]			):
           snake_case__       :   Union[str, Any]	      =    ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # This will move each submodule on different devices
           snake_case__       :   Optional[Any]	      =    0 if torch.cuda.is_available() else 'cpu'
           attach_align_device_hook(__lowerCamelCase     ,							execution_device=__lowerCamelCase     ,							offload=__lowerCamelCase			)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           # Buffers are not included in the offload by default, so are on the execution device
           snake_case__       :   Tuple	      =    torch.device(__lowerCamelCase			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							__lowerCamelCase			)

           snake_case__       :   Any	      =    torch.randn(2     ,							3			)
           snake_case__       :   List[str]	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__lowerCamelCase			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # Now test with buffers included in the offload
           attach_align_device_hook(__lowerCamelCase     ,							execution_device=__lowerCamelCase     ,							offload=__lowerCamelCase     ,							offload_buffers=__lowerCamelCase			)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							torch.device('meta'			)			)

           snake_case__       :   Optional[int]	      =    torch.randn(2     ,							3			)
           snake_case__       :   List[Any]	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__lowerCamelCase			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)





     def 			_lowerCAmelCase      (     self     :					Any			):
           snake_case__       :   Dict	      =    ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # This will move each submodule on different devices
           snake_case__       :   int	      =    0 if torch.cuda.is_available() else 'cpu'
           attach_align_device_hook(
               __lowerCamelCase     ,							execution_device=__lowerCamelCase     ,							offload=__lowerCamelCase     ,							weights_map=model.state_dict()			)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           # Buffers are not included in the offload by default, so are on the execution device
           snake_case__       :   List[str]	      =    torch.device(__lowerCamelCase			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							__lowerCamelCase			)

           snake_case__       :   str	      =    torch.randn(2     ,							3			)
           snake_case__       :   List[Any]	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__lowerCamelCase			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)

           # Now test with buffers included in the offload
           attach_align_device_hook(
               __lowerCamelCase     ,							execution_device=__lowerCamelCase     ,							offload=__lowerCamelCase     ,							weights_map=model.state_dict()     ,							offload_buffers=__lowerCamelCase     ,							)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('meta'			)			)
           self.assertEqual(model.batchnorm.running_mean.device     ,							torch.device('meta'			)			)

           snake_case__       :   List[str]	      =    torch.randn(2     ,							3			)
           snake_case__       :   List[str]	      =    model(__lowerCamelCase			)
           self.assertEqual(output.device     ,							__lowerCamelCase			)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__lowerCamelCase			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.batchnorm.weight.device     ,							torch.device('cpu'			)			)
           self.assertEqual(model.lineara.weight.device     ,							torch.device('cpu'			)			)






'''simple docstring'''






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE		)    ->   str:
      snake_case__      ,							snake_case__       :   Optional[int]	      =    [], []
      while len(__SCREAMING_SNAKE_CASE		) > 1:
            snake_case__      ,							snake_case__       :   Tuple	      =    min(__SCREAMING_SNAKE_CASE		), max(__SCREAMING_SNAKE_CASE		)
            start.append(__SCREAMING_SNAKE_CASE		)
            end.append(__SCREAMING_SNAKE_CASE		)
            collection.remove(__SCREAMING_SNAKE_CASE		)
            collection.remove(__SCREAMING_SNAKE_CASE		)
      end.reverse()
      return start + collection + end


if __name__ == "__main__":
      A_			      =		input("Enter numbers separated by a comma:\n").strip()
      A_			      =		[int(item) for item in user_input.split(",")]
      print(*merge_sort(unsorted), sep=",")
