We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
'function' object has no attribute '__self__' in Pytorch 1.3
After upgrading to Pytorch 1.3.0, my training script crashes with `AttributeError`. Relevant lines from the stack:
```
Traceback (most recent call last):
...
  File "/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py", line 299, in __init__
    super(CosineAnnealingLR, self).__init__(optimizer, last_epoch)
  File "/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py", line 58, in __init__
    self.optimizer.step = with_counter(self.optimizer.step)
  File "/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py", line 40, in with_counter
    instance_ref = weakref.ref(method.__self__)
AttributeError: 'function' object has no attribute '__self__'
```

The `step` method of the Amp optimizer is simply patched with a function, but is not made a method again in https://github.com/NVIDIA/apex/blob/088985936518be7e25795a30d8ab33affa9db6ed/apex/amp/_initialize.py#L235-L245, therefore it does not have a `__self__`.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `apex/amp/_initialize.py`
Content:
```
1 import torch
2 from torch._six import string_classes
3 import functools
4 import numpy as np
5 import sys
6 import warnings
7 from ._amp_state import _amp_state, warn_or_err, container_abcs
8 from .handle import disable_casts
9 from .scaler import LossScaler
10 from ._process_optimizer import _process_optimizer
11 from apex.fp16_utils import convert_network
12 from ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general
13 from ..contrib.optimizers import FP16_Optimizer as FP16_Optimizer_for_fused
14 
15 if torch.distributed.is_available():
16     from ..parallel import DistributedDataParallel as apex_DDP
17     from ..parallel.LARC import LARC
18 
19 
20 def to_type(dtype, t):
21     if isinstance(t, torch.Tensor):
22         if not t.is_cuda:
23             # This should not be a hard error, since it may be legitimate.
24             warnings.warn("An input tensor was not cuda.")
25         # GANs require this.
26         # if t.requires_grad:
27         #     warn_or_err("input data requires grad.  Since input data is not a model parameter,\n"
28         #         "its gradients will not be properly allreduced by DDP.")
29         if t.is_floating_point():
30             return t.to(dtype)
31         return t
32     else:
33         # Trust the user's custom batch type, that's all I can do here.
34         return t.to(dtype)
35 
36 
37 # Modified from torch.optim.optimizer.py.  This is a bit more general than casted_args in utils.py.
38 def applier(value, fn):
39     if isinstance(value, torch.Tensor):
40         return fn(value)
41     elif isinstance(value, string_classes):
42         return value
43     elif isinstance(value, np.ndarray):
44         return value
45     elif hasattr(value, "to"): # Allow handling of custom batch classes
46         return fn(value)
47     elif isinstance(value, container_abcs.Mapping):
48         return {applier(k, fn) : applier(v, fn) for k, v in value.items()}
49     elif isinstance(value, container_abcs.Iterable):
50         return type(value)(applier(v, fn) for v in value)
51     else:
52         # Do I want this to fire off even if someone chooses to pass something ordinary like
53         # an int or float?  May be more annoying than it's worth.
54         # print("Warning:  unrecognized type in applier.  If your input data is a custom class, "
55         #     "provide it with a .to(dtype) method which converts its floating-point Tensors to dtype. "
56         #     "Amp will check for your custom to() and invoke it to cast the batch's "
57         #     "floating-point Tensors to the appropriate type. "
58         #     "Also, if your data is a custom class, it is your responsibility to ensure that "
59         #     "any Tensors you want to be cuda are already cuda."
60         return value
61 
62 
63 def check_models(models):
64     for model in models:
65         parallel_type = None
66         if isinstance(model, torch.nn.parallel.DistributedDataParallel):
67             parallel_type = "torch.nn.parallel.DistributedDataParallel"
68         if ('apex_DDP' in sys.modules) and isinstance(model, apex_DDP):
69             parallel_type = "apex.parallel.DistributedDataParallel"
70         if isinstance(model, torch.nn.parallel.DataParallel):
71             parallel_type = "torch.nn.parallel.DataParallel"
72         if parallel_type is not None:
73             raise RuntimeError("Incoming model is an instance of {}. ".format(parallel_type) +
74                 "Parallel wrappers should only be applied to the model(s) AFTER \n"
75                 "the model(s) have been returned from amp.initialize.")
76 
77 
78 def check_params_fp32(models):
79     for model in models:
80         for name, param in model.named_parameters():
81             if param.is_floating_point():
82                 if 'Half' in param.type():
83                     warn_or_err("Found param {} with type {}, expected torch.cuda.FloatTensor.\n"
84                         "When using amp.initialize, you do not need to call .half() on your model\n"
85                         "before passing it, no matter what optimization level you choose.".format(
86                         name, param.type()))
87                 elif not param.is_cuda:
88                     warn_or_err("Found param {} with type {}, expected torch.cuda.FloatTensor.\n"
89                         "When using amp.initialize, you need to provide a model with parameters\n"
90                         "located on a CUDA device before passing it no matter what optimization level\n"
91                         "you chose. Use model.to('cuda') to use the default device.".format(
92                         name, param.type()))
93 
94         # Backward compatibility for PyTorch 0.4
95         if hasattr(model, 'named_buffers'):
96             buf_iter = model.named_buffers()
97         else:
98             buf_iter = model._buffers
99         for obj in buf_iter:
100             if type(obj)==tuple:
101                 name, buf = obj
102             else:
103                 name, buf = obj, buf_iter[obj]
104             if buf.is_floating_point():
105                 if 'Half' in buf.type():
106                     warn_or_err("Found buffer {} with type {}, expected torch.cuda.FloatTensor.\n"
107                         "When using amp.initialize, you do not need to call .half() on your model\n"
108                         "before passing it, no matter what optimization level you choose.".format(
109                         name, buf.type()))
110                 elif not buf.is_cuda:
111                     warn_or_err("Found buffer {} with type {}, expected torch.cuda.FloatTensor.\n"
112                         "When using amp.initialize, you need to provide a model with buffers\n"
113                         "located on a CUDA device before passing it no matter what optimization level\n"
114                         "you chose. Use model.to('cuda') to use the default device.".format(
115                         name, buf.type()))
116 
117 
118 def check_optimizers(optimizers):
119     for optim in optimizers:
120         bad_optim_type = None
121         if isinstance(optim, FP16_Optimizer_general):
122             bad_optim_type = "apex.fp16_utils.FP16_Optimizer"
123         if isinstance(optim, FP16_Optimizer_for_fused):
124             bad_optim_type = "apex.optimizers.FP16_Optimizer"
125         if bad_optim_type is not None:
126             raise RuntimeError("An incoming optimizer is an instance of {}. ".format(bad_optim_type) +
127                                "The optimizer(s) passed to amp.initialize() must be bare \n"
128                                "instances of either ordinary Pytorch optimizers, or Apex fused \n"
129                                "optimizers.\n")
130 
131 
132 class O2StateDictHook(object):
133     def __init__(self, fn):
134         self.fn = fn
135 
136     def __call__(self, module, state_dict, prefix, local_metadata):
137         for key in state_dict:
138             param = state_dict[key]
139             if 'Half' in param.type():
140                 param = param.to(torch.float32)
141                 state_dict[key] = param
142 
143 
144 def _initialize(models, optimizers, properties, num_losses=1, cast_model_outputs=None):
145     from .amp import init as amp_init
146 
147     optimizers_was_list = False
148     if isinstance(optimizers, torch.optim.Optimizer) or ('LARC' in sys.modules and isinstance(optimizers, LARC)):
149         optimizers = [optimizers]
150     elif optimizers is None:
151         optimizers = []
152     elif isinstance(optimizers, list):
153         optimizers_was_list = True
154         check_optimizers(optimizers)
155     else:
156         check_optimizers([optimizers])
157         raise TypeError("optimizers must be either a single optimizer or a list of optimizers.")
158 
159     if isinstance(models, torch.nn.Module):
160         models_was_list = False
161         models = [models]
162     elif isinstance(models, list):
163         models_was_list = True
164     else:
165         raise TypeError("models must be either a single model or a list of models.")
166 
167     check_models(models)
168 
169     if not _amp_state.allow_incoming_model_not_fp32:
170         check_params_fp32(models)
171 
172     # In the future, when FP16_Optimizer can be deprecated and master weights can
173     # become an attribute, remember to stash master weights before casting the model.
174 
175     if properties.cast_model_type:
176         if properties.keep_batchnorm_fp32:
177             for model in models:
178                 convert_network(model, properties.cast_model_type)
179         else:
180             for model in models:
181                 model.to(properties.cast_model_type)
182 
183         input_caster = functools.partial(to_type, properties.cast_model_type)
184         if cast_model_outputs is not None:
185             output_caster = functools.partial(to_type, cast_model_outputs)
186         else:
187             output_caster = functools.partial(to_type, torch.float32)
188 
189         for model in models:
190             # Patch the forward method to cast incoming data to the correct type, and
191             # outgoing data to float32, so "the user never needs to call .half()."
192             # I like writing things explicitly more than decorators.
193             def patch_forward(old_fwd):
194                 def new_fwd(*args, **kwargs):
195                     output = old_fwd(*applier(args, input_caster),
196                                      **applier(kwargs, input_caster))
197                     return applier(output, output_caster)
198                 return new_fwd
199 
200             model.forward = patch_forward(model.forward)
201 
202         # State dict trick to recast any preexisting per-param state tensors
203         for optimizer in optimizers:
204             optimizer.load_state_dict(optimizer.state_dict())
205 
206         # patch model.state_dict() to return float32 params
207         for model in models:
208             for module in model.modules():
209                 module._register_state_dict_hook(O2StateDictHook(functools.partial(to_type, torch.float32)))
210 
211     elif cast_model_outputs is not None:
212         output_caster = functools.partial(to_type, cast_model_outputs)
213 
214         for model in models:
215             def patch_forward(old_fwd):
216                 def new_fwd(*args, **kwargs):
217                     output = old_fwd(*args, **kwargs)
218                     return applier(output, output_caster)
219                 return new_fwd
220 
221             model.forward = patch_forward(model.forward)
222 
223     for i, optimizer in enumerate(optimizers):
224         optimizers[i] = _process_optimizer(optimizer, properties)
225 
226     _amp_state.loss_scalers = []
227     for _ in range(num_losses):
228         _amp_state.loss_scalers.append(LossScaler(properties.loss_scale,
229                                                   min_loss_scale=_amp_state.min_loss_scale,
230                                                   max_loss_scale=_amp_state.max_loss_scale))
231 
232     if properties.patch_torch_functions:
233         # handle is unused here. It's accessible later through a global value anyway.
234         handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))
235         for optimizer in optimizers:
236             # Disable Amp casting for the optimizer step, because it should only be
237             # applied to FP32 master params anyway.
238             def patch_step(old_step):
239                 def new_step(*args, **kwargs):
240                     with disable_casts():
241                         output = old_step(*args, **kwargs)
242                     return output
243                 return new_step
244 
245             optimizer.step = patch_step(optimizer.step)
246 
247     if optimizers_was_list:
248         if models_was_list:
249             return models, optimizers
250         else:
251             return models[0], optimizers
252     else:
253         if models_was_list:
254             if len(optimizers) == 0:
255                 return models
256             else:
257                 return models, optimizers[0]
258         else:
259             if len(optimizers) == 0:
260                 return models[0]
261             else:
262                 return models[0], optimizers[0]
263
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/apex/amp/_initialize.py b/apex/amp/_initialize.py
--- a/apex/amp/_initialize.py
+++ b/apex/amp/_initialize.py
@@ -3,6 +3,7 @@
 import functools
 import numpy as np
 import sys
+from types import MethodType
 import warnings
 from ._amp_state import _amp_state, warn_or_err, container_abcs
 from .handle import disable_casts
@@ -236,13 +237,13 @@
             # Disable Amp casting for the optimizer step, because it should only be
             # applied to FP32 master params anyway.
             def patch_step(old_step):
-                def new_step(*args, **kwargs):
+                def new_step(self, *args, **kwargs):
                     with disable_casts():
                         output = old_step(*args, **kwargs)
                     return output
                 return new_step
 
-            optimizer.step = patch_step(optimizer.step)
+            optimizer.step = MethodType(patch_step(optimizer.step), optimizer)
 
     if optimizers_was_list:
         if models_was_list:

{"golden_diff": "diff --git a/apex/amp/_initialize.py b/apex/amp/_initialize.py\n--- a/apex/amp/_initialize.py\n+++ b/apex/amp/_initialize.py\n@@ -3,6 +3,7 @@\n import functools\n import numpy as np\n import sys\n+from types import MethodType\n import warnings\n from ._amp_state import _amp_state, warn_or_err, container_abcs\n from .handle import disable_casts\n@@ -236,13 +237,13 @@\n             # Disable Amp casting for the optimizer step, because it should only be\n             # applied to FP32 master params anyway.\n             def patch_step(old_step):\n-                def new_step(*args, **kwargs):\n+                def new_step(self, *args, **kwargs):\n                     with disable_casts():\n                         output = old_step(*args, **kwargs)\n                     return output\n                 return new_step\n \n-            optimizer.step = patch_step(optimizer.step)\n+            optimizer.step = MethodType(patch_step(optimizer.step), optimizer)\n \n     if optimizers_was_list:\n         if models_was_list:\n", "issue": "'function' object has no attribute '__self__' in Pytorch 1.3\nAfter upgrading to Pytorch 1.3.0, my training script crashes with `AttributeError`. Relevant lines from the stack:\r\n```\r\nTraceback (most recent call last):\r\n...\r\n  File \"/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py\", line 299, in __init__\r\n    super(CosineAnnealingLR, self).__init__(optimizer, last_epoch)\r\n  File \"/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py\", line 58, in __init__\r\n    self.optimizer.step = with_counter(self.optimizer.step)\r\n  File \"/usr/lib/python3.7/site-packages/torch/optim/lr_scheduler.py\", line 40, in with_counter\r\n    instance_ref = weakref.ref(method.__self__)\r\nAttributeError: 'function' object has no attribute '__self__'\r\n```\r\n\r\nThe `step` method of the Amp optimizer is simply patched with a function, but is not made a method again in https://github.com/NVIDIA/apex/blob/088985936518be7e25795a30d8ab33affa9db6ed/apex/amp/_initialize.py#L235-L245, therefore it does not have a `__self__`.\n", "before_files": [{"content": "import torch\nfrom torch._six import string_classes\nimport functools\nimport numpy as np\nimport sys\nimport warnings\nfrom ._amp_state import _amp_state, warn_or_err, container_abcs\nfrom .handle import disable_casts\nfrom .scaler import LossScaler\nfrom ._process_optimizer import _process_optimizer\nfrom apex.fp16_utils import convert_network\nfrom ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general\nfrom ..contrib.optimizers import FP16_Optimizer as FP16_Optimizer_for_fused\n\nif torch.distributed.is_available():\n    from ..parallel import DistributedDataParallel as apex_DDP\n    from ..parallel.LARC import LARC\n\n\ndef to_type(dtype, t):\n    if isinstance(t, torch.Tensor):\n        if not t.is_cuda:\n            # This should not be a hard error, since it may be legitimate.\n            warnings.warn(\"An input tensor was not cuda.\")\n        # GANs require this.\n        # if t.requires_grad:\n        #     warn_or_err(\"input data requires grad.  Since input data is not a model parameter,\\n\"\n        #         \"its gradients will not be properly allreduced by DDP.\")\n        if t.is_floating_point():\n            return t.to(dtype)\n        return t\n    else:\n        # Trust the user's custom batch type, that's all I can do here.\n        return t.to(dtype)\n\n\n# Modified from torch.optim.optimizer.py.  This is a bit more general than casted_args in utils.py.\ndef applier(value, fn):\n    if isinstance(value, torch.Tensor):\n        return fn(value)\n    elif isinstance(value, string_classes):\n        return value\n    elif isinstance(value, np.ndarray):\n        return value\n    elif hasattr(value, \"to\"): # Allow handling of custom batch classes\n        return fn(value)\n    elif isinstance(value, container_abcs.Mapping):\n        return {applier(k, fn) : applier(v, fn) for k, v in value.items()}\n    elif isinstance(value, container_abcs.Iterable):\n        return type(value)(applier(v, fn) for v in value)\n    else:\n        # Do I want this to fire off even if someone chooses to pass something ordinary like\n        # an int or float?  May be more annoying than it's worth.\n        # print(\"Warning:  unrecognized type in applier.  If your input data is a custom class, \"\n        #     \"provide it with a .to(dtype) method which converts its floating-point Tensors to dtype. \"\n        #     \"Amp will check for your custom to() and invoke it to cast the batch's \"\n        #     \"floating-point Tensors to the appropriate type. \"\n        #     \"Also, if your data is a custom class, it is your responsibility to ensure that \"\n        #     \"any Tensors you want to be cuda are already cuda.\"\n        return value\n\n\ndef check_models(models):\n    for model in models:\n        parallel_type = None\n        if isinstance(model, torch.nn.parallel.DistributedDataParallel):\n            parallel_type = \"torch.nn.parallel.DistributedDataParallel\"\n        if ('apex_DDP' in sys.modules) and isinstance(model, apex_DDP):\n            parallel_type = \"apex.parallel.DistributedDataParallel\"\n        if isinstance(model, torch.nn.parallel.DataParallel):\n            parallel_type = \"torch.nn.parallel.DataParallel\"\n        if parallel_type is not None:\n            raise RuntimeError(\"Incoming model is an instance of {}. \".format(parallel_type) +\n                \"Parallel wrappers should only be applied to the model(s) AFTER \\n\"\n                \"the model(s) have been returned from amp.initialize.\")\n\n\ndef check_params_fp32(models):\n    for model in models:\n        for name, param in model.named_parameters():\n            if param.is_floating_point():\n                if 'Half' in param.type():\n                    warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n                        \"before passing it, no matter what optimization level you choose.\".format(\n                        name, param.type()))\n                elif not param.is_cuda:\n                    warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you need to provide a model with parameters\\n\"\n                        \"located on a CUDA device before passing it no matter what optimization level\\n\"\n                        \"you chose. Use model.to('cuda') to use the default device.\".format(\n                        name, param.type()))\n\n        # Backward compatibility for PyTorch 0.4\n        if hasattr(model, 'named_buffers'):\n            buf_iter = model.named_buffers()\n        else:\n            buf_iter = model._buffers\n        for obj in buf_iter:\n            if type(obj)==tuple:\n                name, buf = obj\n            else:\n                name, buf = obj, buf_iter[obj]\n            if buf.is_floating_point():\n                if 'Half' in buf.type():\n                    warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n                        \"before passing it, no matter what optimization level you choose.\".format(\n                        name, buf.type()))\n                elif not buf.is_cuda:\n                    warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you need to provide a model with buffers\\n\"\n                        \"located on a CUDA device before passing it no matter what optimization level\\n\"\n                        \"you chose. Use model.to('cuda') to use the default device.\".format(\n                        name, buf.type()))\n\n\ndef check_optimizers(optimizers):\n    for optim in optimizers:\n        bad_optim_type = None\n        if isinstance(optim, FP16_Optimizer_general):\n            bad_optim_type = \"apex.fp16_utils.FP16_Optimizer\"\n        if isinstance(optim, FP16_Optimizer_for_fused):\n            bad_optim_type = \"apex.optimizers.FP16_Optimizer\"\n        if bad_optim_type is not None:\n            raise RuntimeError(\"An incoming optimizer is an instance of {}. \".format(bad_optim_type) +\n                               \"The optimizer(s) passed to amp.initialize() must be bare \\n\"\n                               \"instances of either ordinary Pytorch optimizers, or Apex fused \\n\"\n                               \"optimizers.\\n\")\n\n\nclass O2StateDictHook(object):\n    def __init__(self, fn):\n        self.fn = fn\n\n    def __call__(self, module, state_dict, prefix, local_metadata):\n        for key in state_dict:\n            param = state_dict[key]\n            if 'Half' in param.type():\n                param = param.to(torch.float32)\n                state_dict[key] = param\n\n\ndef _initialize(models, optimizers, properties, num_losses=1, cast_model_outputs=None):\n    from .amp import init as amp_init\n\n    optimizers_was_list = False\n    if isinstance(optimizers, torch.optim.Optimizer) or ('LARC' in sys.modules and isinstance(optimizers, LARC)):\n        optimizers = [optimizers]\n    elif optimizers is None:\n        optimizers = []\n    elif isinstance(optimizers, list):\n        optimizers_was_list = True\n        check_optimizers(optimizers)\n    else:\n        check_optimizers([optimizers])\n        raise TypeError(\"optimizers must be either a single optimizer or a list of optimizers.\")\n\n    if isinstance(models, torch.nn.Module):\n        models_was_list = False\n        models = [models]\n    elif isinstance(models, list):\n        models_was_list = True\n    else:\n        raise TypeError(\"models must be either a single model or a list of models.\")\n\n    check_models(models)\n\n    if not _amp_state.allow_incoming_model_not_fp32:\n        check_params_fp32(models)\n\n    # In the future, when FP16_Optimizer can be deprecated and master weights can\n    # become an attribute, remember to stash master weights before casting the model.\n\n    if properties.cast_model_type:\n        if properties.keep_batchnorm_fp32:\n            for model in models:\n                convert_network(model, properties.cast_model_type)\n        else:\n            for model in models:\n                model.to(properties.cast_model_type)\n\n        input_caster = functools.partial(to_type, properties.cast_model_type)\n        if cast_model_outputs is not None:\n            output_caster = functools.partial(to_type, cast_model_outputs)\n        else:\n            output_caster = functools.partial(to_type, torch.float32)\n\n        for model in models:\n            # Patch the forward method to cast incoming data to the correct type, and\n            # outgoing data to float32, so \"the user never needs to call .half().\"\n            # I like writing things explicitly more than decorators.\n            def patch_forward(old_fwd):\n                def new_fwd(*args, **kwargs):\n                    output = old_fwd(*applier(args, input_caster),\n                                     **applier(kwargs, input_caster))\n                    return applier(output, output_caster)\n                return new_fwd\n\n            model.forward = patch_forward(model.forward)\n\n        # State dict trick to recast any preexisting per-param state tensors\n        for optimizer in optimizers:\n            optimizer.load_state_dict(optimizer.state_dict())\n\n        # patch model.state_dict() to return float32 params\n        for model in models:\n            for module in model.modules():\n                module._register_state_dict_hook(O2StateDictHook(functools.partial(to_type, torch.float32)))\n\n    elif cast_model_outputs is not None:\n        output_caster = functools.partial(to_type, cast_model_outputs)\n\n        for model in models:\n            def patch_forward(old_fwd):\n                def new_fwd(*args, **kwargs):\n                    output = old_fwd(*args, **kwargs)\n                    return applier(output, output_caster)\n                return new_fwd\n\n            model.forward = patch_forward(model.forward)\n\n    for i, optimizer in enumerate(optimizers):\n        optimizers[i] = _process_optimizer(optimizer, properties)\n\n    _amp_state.loss_scalers = []\n    for _ in range(num_losses):\n        _amp_state.loss_scalers.append(LossScaler(properties.loss_scale,\n                                                  min_loss_scale=_amp_state.min_loss_scale,\n                                                  max_loss_scale=_amp_state.max_loss_scale))\n\n    if properties.patch_torch_functions:\n        # handle is unused here. It's accessible later through a global value anyway.\n        handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))\n        for optimizer in optimizers:\n            # Disable Amp casting for the optimizer step, because it should only be\n            # applied to FP32 master params anyway.\n            def patch_step(old_step):\n                def new_step(*args, **kwargs):\n                    with disable_casts():\n                        output = old_step(*args, **kwargs)\n                    return output\n                return new_step\n\n            optimizer.step = patch_step(optimizer.step)\n\n    if optimizers_was_list:\n        if models_was_list:\n            return models, optimizers\n        else:\n            return models[0], optimizers\n    else:\n        if models_was_list:\n            if len(optimizers) == 0:\n                return models\n            else:\n                return models, optimizers[0]\n        else:\n            if len(optimizers) == 0:\n                return models[0]\n            else:\n                return models[0], optimizers[0]\n", "path": "apex/amp/_initialize.py"}], "after_files": [{"content": "import torch\nfrom torch._six import string_classes\nimport functools\nimport numpy as np\nimport sys\nfrom types import MethodType\nimport warnings\nfrom ._amp_state import _amp_state, warn_or_err, container_abcs\nfrom .handle import disable_casts\nfrom .scaler import LossScaler\nfrom ._process_optimizer import _process_optimizer\nfrom apex.fp16_utils import convert_network\nfrom ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general\nfrom ..contrib.optimizers import FP16_Optimizer as FP16_Optimizer_for_fused\n\nif torch.distributed.is_available():\n    from ..parallel import DistributedDataParallel as apex_DDP\n    from ..parallel.LARC import LARC\n\n\ndef to_type(dtype, t):\n    if isinstance(t, torch.Tensor):\n        if not t.is_cuda:\n            # This should not be a hard error, since it may be legitimate.\n            warnings.warn(\"An input tensor was not cuda.\")\n        # GANs require this.\n        # if t.requires_grad:\n        #     warn_or_err(\"input data requires grad.  Since input data is not a model parameter,\\n\"\n        #         \"its gradients will not be properly allreduced by DDP.\")\n        if t.is_floating_point():\n            return t.to(dtype)\n        return t\n    else:\n        # Trust the user's custom batch type, that's all I can do here.\n        return t.to(dtype)\n\n\n# Modified from torch.optim.optimizer.py.  This is a bit more general than casted_args in utils.py.\ndef applier(value, fn):\n    if isinstance(value, torch.Tensor):\n        return fn(value)\n    elif isinstance(value, string_classes):\n        return value\n    elif isinstance(value, np.ndarray):\n        return value\n    elif hasattr(value, \"to\"): # Allow handling of custom batch classes\n        return fn(value)\n    elif isinstance(value, container_abcs.Mapping):\n        return {applier(k, fn) : applier(v, fn) for k, v in value.items()}\n    elif isinstance(value, container_abcs.Iterable):\n        return type(value)(applier(v, fn) for v in value)\n    else:\n        # Do I want this to fire off even if someone chooses to pass something ordinary like\n        # an int or float?  May be more annoying than it's worth.\n        # print(\"Warning:  unrecognized type in applier.  If your input data is a custom class, \"\n        #     \"provide it with a .to(dtype) method which converts its floating-point Tensors to dtype. \"\n        #     \"Amp will check for your custom to() and invoke it to cast the batch's \"\n        #     \"floating-point Tensors to the appropriate type. \"\n        #     \"Also, if your data is a custom class, it is your responsibility to ensure that \"\n        #     \"any Tensors you want to be cuda are already cuda.\"\n        return value\n\n\ndef check_models(models):\n    for model in models:\n        parallel_type = None\n        if isinstance(model, torch.nn.parallel.DistributedDataParallel):\n            parallel_type = \"torch.nn.parallel.DistributedDataParallel\"\n        if ('apex_DDP' in sys.modules) and isinstance(model, apex_DDP):\n            parallel_type = \"apex.parallel.DistributedDataParallel\"\n        if isinstance(model, torch.nn.parallel.DataParallel):\n            parallel_type = \"torch.nn.parallel.DataParallel\"\n        if parallel_type is not None:\n            raise RuntimeError(\"Incoming model is an instance of {}. \".format(parallel_type) +\n                \"Parallel wrappers should only be applied to the model(s) AFTER \\n\"\n                \"the model(s) have been returned from amp.initialize.\")\n\n\ndef check_params_fp32(models):\n    for model in models:\n        for name, param in model.named_parameters():\n            if param.is_floating_point():\n                if 'Half' in param.type():\n                    warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n                        \"before passing it, no matter what optimization level you choose.\".format(\n                        name, param.type()))\n                elif not param.is_cuda:\n                    warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you need to provide a model with parameters\\n\"\n                        \"located on a CUDA device before passing it no matter what optimization level\\n\"\n                        \"you chose. Use model.to('cuda') to use the default device.\".format(\n                        name, param.type()))\n\n        # Backward compatibility for PyTorch 0.4\n        if hasattr(model, 'named_buffers'):\n            buf_iter = model.named_buffers()\n        else:\n            buf_iter = model._buffers\n        for obj in buf_iter:\n            if type(obj)==tuple:\n                name, buf = obj\n            else:\n                name, buf = obj, buf_iter[obj]\n            if buf.is_floating_point():\n                if 'Half' in buf.type():\n                    warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n                        \"before passing it, no matter what optimization level you choose.\".format(\n                        name, buf.type()))\n                elif not buf.is_cuda:\n                    warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n                        \"When using amp.initialize, you need to provide a model with buffers\\n\"\n                        \"located on a CUDA device before passing it no matter what optimization level\\n\"\n                        \"you chose. Use model.to('cuda') to use the default device.\".format(\n                        name, buf.type()))\n\n\ndef check_optimizers(optimizers):\n    for optim in optimizers:\n        bad_optim_type = None\n        if isinstance(optim, FP16_Optimizer_general):\n            bad_optim_type = \"apex.fp16_utils.FP16_Optimizer\"\n        if isinstance(optim, FP16_Optimizer_for_fused):\n            bad_optim_type = \"apex.optimizers.FP16_Optimizer\"\n        if bad_optim_type is not None:\n            raise RuntimeError(\"An incoming optimizer is an instance of {}. \".format(bad_optim_type) +\n                               \"The optimizer(s) passed to amp.initialize() must be bare \\n\"\n                               \"instances of either ordinary Pytorch optimizers, or Apex fused \\n\"\n                               \"optimizers.\\n\")\n\n\nclass O2StateDictHook(object):\n    def __init__(self, fn):\n        self.fn = fn\n\n    def __call__(self, module, state_dict, prefix, local_metadata):\n        for key in state_dict:\n            param = state_dict[key]\n            if 'Half' in param.type():\n                param = param.to(torch.float32)\n                state_dict[key] = param\n\n\ndef _initialize(models, optimizers, properties, num_losses=1, cast_model_outputs=None):\n    from .amp import init as amp_init\n\n    optimizers_was_list = False\n    if isinstance(optimizers, torch.optim.Optimizer) or ('LARC' in sys.modules and isinstance(optimizers, LARC)):\n        optimizers = [optimizers]\n    elif optimizers is None:\n        optimizers = []\n    elif isinstance(optimizers, list):\n        optimizers_was_list = True\n        check_optimizers(optimizers)\n    else:\n        check_optimizers([optimizers])\n        raise TypeError(\"optimizers must be either a single optimizer or a list of optimizers.\")\n\n    if isinstance(models, torch.nn.Module):\n        models_was_list = False\n        models = [models]\n    elif isinstance(models, list):\n        models_was_list = True\n    else:\n        raise TypeError(\"models must be either a single model or a list of models.\")\n\n    check_models(models)\n\n    if not _amp_state.allow_incoming_model_not_fp32:\n        check_params_fp32(models)\n\n    # In the future, when FP16_Optimizer can be deprecated and master weights can\n    # become an attribute, remember to stash master weights before casting the model.\n\n    if properties.cast_model_type:\n        if properties.keep_batchnorm_fp32:\n            for model in models:\n                convert_network(model, properties.cast_model_type)\n        else:\n            for model in models:\n                model.to(properties.cast_model_type)\n\n        input_caster = functools.partial(to_type, properties.cast_model_type)\n        if cast_model_outputs is not None:\n            output_caster = functools.partial(to_type, cast_model_outputs)\n        else:\n            output_caster = functools.partial(to_type, torch.float32)\n\n        for model in models:\n            # Patch the forward method to cast incoming data to the correct type, and\n            # outgoing data to float32, so \"the user never needs to call .half().\"\n            # I like writing things explicitly more than decorators.\n            def patch_forward(old_fwd):\n                def new_fwd(*args, **kwargs):\n                    output = old_fwd(*applier(args, input_caster),\n                                     **applier(kwargs, input_caster))\n                    return applier(output, output_caster)\n                return new_fwd\n\n            model.forward = patch_forward(model.forward)\n\n        # State dict trick to recast any preexisting per-param state tensors\n        for optimizer in optimizers:\n            optimizer.load_state_dict(optimizer.state_dict())\n\n        # patch model.state_dict() to return float32 params\n        for model in models:\n            for module in model.modules():\n                module._register_state_dict_hook(O2StateDictHook(functools.partial(to_type, torch.float32)))\n\n    elif cast_model_outputs is not None:\n        output_caster = functools.partial(to_type, cast_model_outputs)\n\n        for model in models:\n            def patch_forward(old_fwd):\n                def new_fwd(*args, **kwargs):\n                    output = old_fwd(*args, **kwargs)\n                    return applier(output, output_caster)\n                return new_fwd\n\n            model.forward = patch_forward(model.forward)\n\n    for i, optimizer in enumerate(optimizers):\n        optimizers[i] = _process_optimizer(optimizer, properties)\n\n    _amp_state.loss_scalers = []\n    for _ in range(num_losses):\n        _amp_state.loss_scalers.append(LossScaler(properties.loss_scale,\n                                                  min_loss_scale=_amp_state.min_loss_scale,\n                                                  max_loss_scale=_amp_state.max_loss_scale))\n\n    if properties.patch_torch_functions:\n        # handle is unused here. It's accessible later through a global value anyway.\n        handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))\n        for optimizer in optimizers:\n            # Disable Amp casting for the optimizer step, because it should only be\n            # applied to FP32 master params anyway.\n            def patch_step(old_step):\n                def new_step(self, *args, **kwargs):\n                    with disable_casts():\n                        output = old_step(*args, **kwargs)\n                    return output\n                return new_step\n\n            optimizer.step = MethodType(patch_step(optimizer.step), optimizer)\n\n    if optimizers_was_list:\n        if models_was_list:\n            return models, optimizers\n        else:\n            return models[0], optimizers\n    else:\n        if models_was_list:\n            if len(optimizers) == 0:\n                return models\n            else:\n                return models, optimizers[0]\n        else:\n            if len(optimizers) == 0:\n                return models[0]\n            else:\n                return models[0], optimizers[0]\n", "path": "apex/amp/_initialize.py"}]}