We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
❓ Warning : This overload of addcdiv_ is deprecated
# ❓ Questions & Help

When running the [official Colab example of GLUE](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/trainer/01_text_classification.ipynb), during training I receive a `UserWarning` :

```
/pytorch/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of addcdiv_ is deprecated:
	addcdiv_(Number value, Tensor tensor1, Tensor tensor2)
Consider using one of the following signatures instead:
	addcdiv_(Tensor tensor1, Tensor tensor2, *, Number value)
```

---

**Is it expected ?**



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/transformers/optimization.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """PyTorch optimization for BERT model."""
16 
17 import logging
18 import math
19 
20 import torch
21 from torch.optim import Optimizer
22 from torch.optim.lr_scheduler import LambdaLR
23 
24 
25 logger = logging.getLogger(__name__)
26 
27 
28 def get_constant_schedule(optimizer, last_epoch=-1):
29     """ Create a schedule with a constant learning rate.
30     """
31     return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch)
32 
33 
34 def get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1):
35     """ Create a schedule with a constant learning rate preceded by a warmup
36     period during which the learning rate increases linearly between 0 and 1.
37     """
38 
39     def lr_lambda(current_step):
40         if current_step < num_warmup_steps:
41             return float(current_step) / float(max(1.0, num_warmup_steps))
42         return 1.0
43 
44     return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch)
45 
46 
47 def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):
48     """ Create a schedule with a learning rate that decreases linearly after
49     linearly increasing during a warmup period.
50     """
51 
52     def lr_lambda(current_step):
53         if current_step < num_warmup_steps:
54             return float(current_step) / float(max(1, num_warmup_steps))
55         return max(
56             0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))
57         )
58 
59     return LambdaLR(optimizer, lr_lambda, last_epoch)
60 
61 
62 def get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=0.5, last_epoch=-1):
63     """ Create a schedule with a learning rate that decreases following the
64     values of the cosine function between 0 and `pi * cycles` after a warmup
65     period during which it increases linearly between 0 and 1.
66     """
67 
68     def lr_lambda(current_step):
69         if current_step < num_warmup_steps:
70             return float(current_step) / float(max(1, num_warmup_steps))
71         progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))
72         return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress)))
73 
74     return LambdaLR(optimizer, lr_lambda, last_epoch)
75 
76 
77 def get_cosine_with_hard_restarts_schedule_with_warmup(
78     optimizer, num_warmup_steps, num_training_steps, num_cycles=1.0, last_epoch=-1
79 ):
80     """ Create a schedule with a learning rate that decreases following the
81     values of the cosine function with several hard restarts, after a warmup
82     period during which it increases linearly between 0 and 1.
83     """
84 
85     def lr_lambda(current_step):
86         if current_step < num_warmup_steps:
87             return float(current_step) / float(max(1, num_warmup_steps))
88         progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))
89         if progress >= 1.0:
90             return 0.0
91         return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(num_cycles) * progress) % 1.0))))
92 
93     return LambdaLR(optimizer, lr_lambda, last_epoch)
94 
95 
96 class AdamW(Optimizer):
97     """ Implements Adam algorithm with weight decay fix.
98 
99     Parameters:
100         lr (float): learning rate. Default 1e-3.
101         betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999)
102         eps (float): Adams epsilon. Default: 1e-6
103         weight_decay (float): Weight decay. Default: 0.0
104         correct_bias (bool): can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository). Default True.
105     """
106 
107     def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True):
108         if lr < 0.0:
109             raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr))
110         if not 0.0 <= betas[0] < 1.0:
111             raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0]))
112         if not 0.0 <= betas[1] < 1.0:
113             raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1]))
114         if not 0.0 <= eps:
115             raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps))
116         defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, correct_bias=correct_bias)
117         super().__init__(params, defaults)
118 
119     def step(self, closure=None):
120         """Performs a single optimization step.
121 
122         Arguments:
123             closure (callable, optional): A closure that reevaluates the model
124                 and returns the loss.
125         """
126         loss = None
127         if closure is not None:
128             loss = closure()
129 
130         for group in self.param_groups:
131             for p in group["params"]:
132                 if p.grad is None:
133                     continue
134                 grad = p.grad.data
135                 if grad.is_sparse:
136                     raise RuntimeError("Adam does not support sparse gradients, please consider SparseAdam instead")
137 
138                 state = self.state[p]
139 
140                 # State initialization
141                 if len(state) == 0:
142                     state["step"] = 0
143                     # Exponential moving average of gradient values
144                     state["exp_avg"] = torch.zeros_like(p.data)
145                     # Exponential moving average of squared gradient values
146                     state["exp_avg_sq"] = torch.zeros_like(p.data)
147 
148                 exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
149                 beta1, beta2 = group["betas"]
150 
151                 state["step"] += 1
152 
153                 # Decay the first and second moment running average coefficient
154                 # In-place operations to update the averages at the same time
155                 exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)
156                 exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2)
157                 denom = exp_avg_sq.sqrt().add_(group["eps"])
158 
159                 step_size = group["lr"]
160                 if group["correct_bias"]:  # No bias correction for Bert
161                     bias_correction1 = 1.0 - beta1 ** state["step"]
162                     bias_correction2 = 1.0 - beta2 ** state["step"]
163                     step_size = step_size * math.sqrt(bias_correction2) / bias_correction1
164 
165                 p.data.addcdiv_(-step_size, exp_avg, denom)
166 
167                 # Just adding the square of the weights to the loss function is *not*
168                 # the correct way of using L2 regularization/weight decay with Adam,
169                 # since that will interact with the m and v parameters in strange ways.
170                 #
171                 # Instead we want to decay the weights in a manner that doesn't interact
172                 # with the m/v parameters. This is equivalent to adding the square
173                 # of the weights to the loss with plain (non-momentum) SGD.
174                 # Add weight decay at the end (fixed version)
175                 if group["weight_decay"] > 0.0:
176                     p.data.add_(p.data, alpha=-group["lr"] * group["weight_decay"])
177 
178         return loss
179
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/transformers/optimization.py b/src/transformers/optimization.py
--- a/src/transformers/optimization.py
+++ b/src/transformers/optimization.py
@@ -162,7 +162,7 @@
                     bias_correction2 = 1.0 - beta2 ** state["step"]
                     step_size = step_size * math.sqrt(bias_correction2) / bias_correction1
 
-                p.data.addcdiv_(-step_size, exp_avg, denom)
+                p.data.addcdiv_(exp_avg, denom, value=-step_size)
 
                 # Just adding the square of the weights to the loss function is *not*
                 # the correct way of using L2 regularization/weight decay with Adam,

{"golden_diff": "diff --git a/src/transformers/optimization.py b/src/transformers/optimization.py\n--- a/src/transformers/optimization.py\n+++ b/src/transformers/optimization.py\n@@ -162,7 +162,7 @@\n                     bias_correction2 = 1.0 - beta2 ** state[\"step\"]\n                     step_size = step_size * math.sqrt(bias_correction2) / bias_correction1\n \n-                p.data.addcdiv_(-step_size, exp_avg, denom)\n+                p.data.addcdiv_(exp_avg, denom, value=-step_size)\n \n                 # Just adding the square of the weights to the loss function is *not*\n                 # the correct way of using L2 regularization/weight decay with Adam,\n", "issue": "\u2753 Warning : This overload of addcdiv_ is deprecated\n# \u2753 Questions & Help\r\n\r\nWhen running the [official Colab example of GLUE](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/trainer/01_text_classification.ipynb), during training I receive a `UserWarning` :\r\n\r\n```\r\n/pytorch/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of addcdiv_ is deprecated:\r\n\taddcdiv_(Number value, Tensor tensor1, Tensor tensor2)\r\nConsider using one of the following signatures instead:\r\n\taddcdiv_(Tensor tensor1, Tensor tensor2, *, Number value)\r\n```\r\n\r\n---\r\n\r\n**Is it expected ?**\r\n\r\n\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"PyTorch optimization for BERT model.\"\"\"\n\nimport logging\nimport math\n\nimport torch\nfrom torch.optim import Optimizer\nfrom torch.optim.lr_scheduler import LambdaLR\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_constant_schedule(optimizer, last_epoch=-1):\n    \"\"\" Create a schedule with a constant learning rate.\n    \"\"\"\n    return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch)\n\n\ndef get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1):\n    \"\"\" Create a schedule with a constant learning rate preceded by a warmup\n    period during which the learning rate increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1.0, num_warmup_steps))\n        return 1.0\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch)\n\n\ndef get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):\n    \"\"\" Create a schedule with a learning rate that decreases linearly after\n    linearly increasing during a warmup period.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        return max(\n            0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))\n        )\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=0.5, last_epoch=-1):\n    \"\"\" Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress)))\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef get_cosine_with_hard_restarts_schedule_with_warmup(\n    optimizer, num_warmup_steps, num_training_steps, num_cycles=1.0, last_epoch=-1\n):\n    \"\"\" Create a schedule with a learning rate that decreases following the\n    values of the cosine function with several hard restarts, after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        if progress >= 1.0:\n            return 0.0\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(num_cycles) * progress) % 1.0))))\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\nclass AdamW(Optimizer):\n    \"\"\" Implements Adam algorithm with weight decay fix.\n\n    Parameters:\n        lr (float): learning rate. Default 1e-3.\n        betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999)\n        eps (float): Adams epsilon. Default: 1e-6\n        weight_decay (float): Weight decay. Default: 0.0\n        correct_bias (bool): can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository). Default True.\n    \"\"\"\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True):\n        if lr < 0.0:\n            raise ValueError(\"Invalid learning rate: {} - should be >= 0.0\".format(lr))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter: {} - should be in [0.0, 1.0[\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter: {} - should be in [0.0, 1.0[\".format(betas[1]))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {} - should be >= 0.0\".format(eps))\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, correct_bias=correct_bias)\n        super().__init__(params, defaults)\n\n    def step(self, closure=None):\n        \"\"\"Performs a single optimization step.\n\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group[\"params\"]:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                if grad.is_sparse:\n                    raise RuntimeError(\"Adam does not support sparse gradients, please consider SparseAdam instead\")\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state[\"step\"] = 0\n                    # Exponential moving average of gradient values\n                    state[\"exp_avg\"] = torch.zeros_like(p.data)\n                    # Exponential moving average of squared gradient values\n                    state[\"exp_avg_sq\"] = torch.zeros_like(p.data)\n\n                exp_avg, exp_avg_sq = state[\"exp_avg\"], state[\"exp_avg_sq\"]\n                beta1, beta2 = group[\"betas\"]\n\n                state[\"step\"] += 1\n\n                # Decay the first and second moment running average coefficient\n                # In-place operations to update the averages at the same time\n                exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2)\n                denom = exp_avg_sq.sqrt().add_(group[\"eps\"])\n\n                step_size = group[\"lr\"]\n                if group[\"correct_bias\"]:  # No bias correction for Bert\n                    bias_correction1 = 1.0 - beta1 ** state[\"step\"]\n                    bias_correction2 = 1.0 - beta2 ** state[\"step\"]\n                    step_size = step_size * math.sqrt(bias_correction2) / bias_correction1\n\n                p.data.addcdiv_(-step_size, exp_avg, denom)\n\n                # Just adding the square of the weights to the loss function is *not*\n                # the correct way of using L2 regularization/weight decay with Adam,\n                # since that will interact with the m and v parameters in strange ways.\n                #\n                # Instead we want to decay the weights in a manner that doesn't interact\n                # with the m/v parameters. This is equivalent to adding the square\n                # of the weights to the loss with plain (non-momentum) SGD.\n                # Add weight decay at the end (fixed version)\n                if group[\"weight_decay\"] > 0.0:\n                    p.data.add_(p.data, alpha=-group[\"lr\"] * group[\"weight_decay\"])\n\n        return loss\n", "path": "src/transformers/optimization.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"PyTorch optimization for BERT model.\"\"\"\n\nimport logging\nimport math\n\nimport torch\nfrom torch.optim import Optimizer\nfrom torch.optim.lr_scheduler import LambdaLR\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_constant_schedule(optimizer, last_epoch=-1):\n    \"\"\" Create a schedule with a constant learning rate.\n    \"\"\"\n    return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch)\n\n\ndef get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1):\n    \"\"\" Create a schedule with a constant learning rate preceded by a warmup\n    period during which the learning rate increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1.0, num_warmup_steps))\n        return 1.0\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch)\n\n\ndef get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):\n    \"\"\" Create a schedule with a learning rate that decreases linearly after\n    linearly increasing during a warmup period.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        return max(\n            0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))\n        )\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=0.5, last_epoch=-1):\n    \"\"\" Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress)))\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef get_cosine_with_hard_restarts_schedule_with_warmup(\n    optimizer, num_warmup_steps, num_training_steps, num_cycles=1.0, last_epoch=-1\n):\n    \"\"\" Create a schedule with a learning rate that decreases following the\n    values of the cosine function with several hard restarts, after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lr_lambda(current_step):\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        if progress >= 1.0:\n            return 0.0\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(num_cycles) * progress) % 1.0))))\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\nclass AdamW(Optimizer):\n    \"\"\" Implements Adam algorithm with weight decay fix.\n\n    Parameters:\n        lr (float): learning rate. Default 1e-3.\n        betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999)\n        eps (float): Adams epsilon. Default: 1e-6\n        weight_decay (float): Weight decay. Default: 0.0\n        correct_bias (bool): can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository). Default True.\n    \"\"\"\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True):\n        if lr < 0.0:\n            raise ValueError(\"Invalid learning rate: {} - should be >= 0.0\".format(lr))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter: {} - should be in [0.0, 1.0[\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter: {} - should be in [0.0, 1.0[\".format(betas[1]))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {} - should be >= 0.0\".format(eps))\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, correct_bias=correct_bias)\n        super().__init__(params, defaults)\n\n    def step(self, closure=None):\n        \"\"\"Performs a single optimization step.\n\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group[\"params\"]:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                if grad.is_sparse:\n                    raise RuntimeError(\"Adam does not support sparse gradients, please consider SparseAdam instead\")\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state[\"step\"] = 0\n                    # Exponential moving average of gradient values\n                    state[\"exp_avg\"] = torch.zeros_like(p.data)\n                    # Exponential moving average of squared gradient values\n                    state[\"exp_avg_sq\"] = torch.zeros_like(p.data)\n\n                exp_avg, exp_avg_sq = state[\"exp_avg\"], state[\"exp_avg_sq\"]\n                beta1, beta2 = group[\"betas\"]\n\n                state[\"step\"] += 1\n\n                # Decay the first and second moment running average coefficient\n                # In-place operations to update the averages at the same time\n                exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2)\n                denom = exp_avg_sq.sqrt().add_(group[\"eps\"])\n\n                step_size = group[\"lr\"]\n                if group[\"correct_bias\"]:  # No bias correction for Bert\n                    bias_correction1 = 1.0 - beta1 ** state[\"step\"]\n                    bias_correction2 = 1.0 - beta2 ** state[\"step\"]\n                    step_size = step_size * math.sqrt(bias_correction2) / bias_correction1\n\n                p.data.addcdiv_(exp_avg, denom, value=-step_size)\n\n                # Just adding the square of the weights to the loss function is *not*\n                # the correct way of using L2 regularization/weight decay with Adam,\n                # since that will interact with the m and v parameters in strange ways.\n                #\n                # Instead we want to decay the weights in a manner that doesn't interact\n                # with the m/v parameters. This is equivalent to adding the square\n                # of the weights to the loss with plain (non-momentum) SGD.\n                # Add weight decay at the end (fixed version)\n                if group[\"weight_decay\"] > 0.0:\n                    p.data.add_(p.data, alpha=-group[\"lr\"] * group[\"weight_decay\"])\n\n        return loss\n", "path": "src/transformers/optimization.py"}]}