text: # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import unittest
  import paddle
  import random
  import numpy as np
  import paddle.fluid as fluid
  from op_test import OpTest
  from functools import partial
  from paddle.framework import core
  from paddle.fluid.framework import _test_eager_guard
  
  
  def adamw_step(inputs, attributes):
      param = inputs['Param']
      grad = inputs['Grad']
      moment1 = inputs['Moment1']
      moment2 = inputs['Moment2']
      lr = inputs['LearningRate']
      beta1_pow = inputs['Beta1Pow']
      beta2_pow = inputs['Beta2Pow']
  
      epsilon = attributes['epsilon']
  
      if 'lr_ratio' in attributes:
          lr = lr * attributes['lr_ratio']
  
      if attributes["with_decay"]:
          coeff = attributes["coeff"]
          decay = 1.0 - lr * coeff
          param2 = param * decay
          param = param2.copy()
  
      if 'beta1' in attributes:
          beta1 = attributes['beta1']
      else:
          beta1 = inputs['Beta1Tensor'][0]
      if 'beta2' in attributes:
          beta2 = attributes['beta2']
      else:
          beta2 = inputs['Beta2Tensor'][0]
  
      moment1_out = beta1 * moment1 + (1 - beta1) * grad
      moment2_out = beta2 * moment2 + (1 - beta2) * np.square(grad)
      lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow)
      param_out = param - lr_t * (moment1_out / (np.sqrt(moment2_out) + epsilon))
      return param_out, moment1_out, moment2_out
  
  
  class TestAdamW(OpTest):
      def setUp(self):
          '''Test AdamW Op with supplied attributes
          '''
          self.op_type = "adamw"
          param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
          grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
          moment1 = np.random.uniform(-1, 1, (102, 105)).astype("float32")
          # The second moment is positive
          moment2 = np.random.random((102, 105)).astype("float32")
  
          learning_rate = 0.004
          beta1 = 0.78
          beta2 = 0.836
          epsilon = 1e-4
          beta1_pow = beta1**10
          beta2_pow = beta2**10
  
          self.inputs = {
              'Param': param,
              'Grad': grad,
              'Moment1': moment1,
              'Moment2': moment2,
              'LearningRate': np.array([learning_rate]).astype("float32"),
              'Beta1Pow': np.array([beta1_pow]).astype("float32"),
              'Beta2Pow': np.array([beta2_pow]).astype("float32")
          }
  
          self.attrs = {
              'epsilon': epsilon,
              'beta1': beta1,
              'beta2': beta2,
              "coeff": 0.5,
              "with_decay": True
          }
  
          param_out, moment1_out, \
              moment2_out = adamw_step(self.inputs, self.attrs)
  
          self.outputs = {
              'Moment1Out': moment1_out,
              'Moment2Out': moment2_out,
              'ParamOut': param_out,
              'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1,
              'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2
          }
  
      def test_check_output(self):
          self.check_output()
  
  
  @unittest.skipIf(not core.is_compiled_with_cuda(),
                   "core is not compiled with CUDA")
  class TestAdamW2(OpTest):
      def setUp(self):
          '''Test AdamW Op with supplied attributes
          '''
          self.op_type = "adamw"
          param = np.random.uniform(-1, 1, (2, 2)).astype("float32")
          grad = np.random.uniform(-1, 1, (2, 2)).astype("float32")
          moment1 = np.random.uniform(-1, 1, (2, 2)).astype("float32")
          # The second moment is positive
          moment2 = np.random.random((2, 2)).astype("float32")
  
          learning_rate = 0.004
          beta1 = 0.78
          beta2 = 0.836
          epsilon = 1e-4
          beta1_pow = beta1**10
          beta2_pow = beta2**10
  
          self.inputs = {
              'Param': param,
              'Grad': grad,
              'Moment1': moment1,
              'Moment2': moment2,
              'LearningRate': np.array([learning_rate]).astype("float32"),
              'Beta1Pow': np.array([beta1_pow]).astype("float32"),
              'Beta2Pow': np.array([beta2_pow]).astype("float32")
          }
  
          self.attrs = {
              'epsilon': epsilon,
              'beta1': beta1,
              'beta2': beta2,
              "lr_ratio": 0.1,
              "coeff": 0.5,
              "with_decay": True
          }
  
          param_out, moment1_out, moment2_out = adamw_step(self.inputs,
                                                           self.attrs)
  
          self.outputs = {
              'Moment1Out': moment1_out,
              'Moment2Out': moment2_out,
              'ParamOut': param_out,
              'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1,
              'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2
          }
  
      def test_check_output(self):
          self.check_output_with_place(core.CUDAPlace(0))
  
  
  class TestAdamWOp(unittest.TestCase):
      def test_adamw_op_dygraph(self):
          paddle.disable_static()
          value = np.arange(26).reshape(2, 13).astype("float32")
          a = paddle.to_tensor(value)
          linear = paddle.nn.Linear(13, 5)
          adam = paddle.optimizer.AdamW(
              learning_rate=0.01,
              parameters=linear.parameters(),
              apply_decay_param_fun=lambda name: True,
              weight_decay=0.01)
  
          for _ in range(2):
              out = linear(a)
              out.backward()
              adam.step()
              adam.clear_gradients()
  
      def test_adamw_op_coverage(self):
          paddle.disable_static()
          value = np.arange(26).reshape(2, 13).astype("float32")
          a = paddle.to_tensor(value)
          linear = paddle.nn.Linear(13, 5)
          adam = paddle.optimizer.AdamW(
              learning_rate=0.0,
              parameters=linear.parameters(),
              apply_decay_param_fun=lambda name: True,
              weight_decay=0.01)
          assert (adam.__str__() is not None)
  
      def test_adamw_op(self):
          paddle.enable_static()
          place = fluid.CPUPlace()
          shape = [2, 3, 8, 8]
          exe = fluid.Executor(place)
          train_prog = fluid.Program()
          startup = fluid.Program()
          with fluid.program_guard(train_prog, startup):
              with fluid.unique_name.guard():
                  data = fluid.data(name="data", shape=shape)
                  conv = fluid.layers.conv2d(data, 8, 3)
                  loss = paddle.mean(conv)
  
                  beta1 = fluid.layers.create_global_var(
                      shape=[1], value=0.85, dtype='float32', persistable=True)
                  beta2 = fluid.layers.create_global_var(
                      shape=[1], value=0.95, dtype='float32', persistable=True)
                  betas = [beta1, beta2]
                  opt = paddle.optimizer.AdamW(
                      learning_rate=1e-5,
                      beta1=beta1,
                      beta2=beta2,
                      weight_decay=0.01,
                      epsilon=1e-8)
                  opt.minimize(loss)
  
          exe.run(startup)
          data_np = np.random.random(shape).astype('float32')
          rets = exe.run(train_prog, feed={"data": data_np}, fetch_list=[loss])
          assert rets[0] is not None
          paddle.disable_static()
  
      def test_adamw_op_invalid_input(self):
          paddle.disable_static()
          linear = paddle.nn.Linear(10, 10)
          with self.assertRaises(ValueError):
              adam = paddle.optimizer.AdamW(
                  0.1, beta1=-1, parameters=linear.parameters())
          with self.assertRaises(ValueError):
              adam = paddle.optimizer.AdamW(
                  0.1, beta2=-1, parameters=linear.parameters())
          with self.assertRaises(ValueError):
              adam = paddle.optimizer.AdamW(
                  0.1, epsilon=-1, parameters=linear.parameters())
  
      def test_api_eager_dygraph(self):
          with _test_eager_guard():
              self.test_adamw_op_dygraph()
              self.test_adamw_op_invalid_input()
  
  
  class TestAdamWOpGroup(TestAdamWOp):
      def test_adamw_op_dygraph(self):
          paddle.disable_static()
          value = np.arange(26).reshape(2, 13).astype("float32")
          a = paddle.to_tensor(value)
          linear_1 = paddle.nn.Linear(13, 5)
          linear_2 = paddle.nn.Linear(5, 3)
          adam = paddle.optimizer.AdamW(
              learning_rate=0.01,
              parameters=[{
                  'params': linear_1.parameters()
              }, {
                  'params': linear_2.parameters(),
                  'weight_decay': 0.001
              }],
              apply_decay_param_fun=lambda name: True,
              weight_decay=0.01)
  
          for _ in range(2):
              out = linear_1(a)
              out = linear_2(out)
              out.backward()
              adam.step()
              adam.clear_gradients()
  
  
  class TestAdamWOpGroupWithLR(TestAdamWOp):
      def test_adamw_op_dygraph(self):
          paddle.disable_static()
          value = np.arange(26).reshape(2, 13).astype("float32")
          a = paddle.to_tensor(value)
          linear_1 = paddle.nn.Linear(13, 5)
          linear_2 = paddle.nn.Linear(5, 3)
          adam = paddle.optimizer.AdamW(
              learning_rate=paddle.optimizer.lr.PiecewiseDecay(
                  boundaries=[3, 6], values=[0.1, 0.2, 0.3]),
              parameters=[{
                  'params': linear_1.parameters(),
                  'learning_rate': 0.1,
              }, {
                  'params': linear_2.parameters(),
                  'weight_decay': 0.001,
              }],
              apply_decay_param_fun=lambda name: True,
              weight_decay=0.01)
  
          for _ in range(2):
              out = linear_1(a)
              out = linear_2(out)
              out.backward()
              adam.step()
              adam.clear_gradients()
  
  
  def simple_lr_setting(param, decay_rate, n_layers):
      if "fc_0" in param.name or "linear_1" in param.name:
          depth = int(param.name.split("_")[2]) + 1
      elif "fc_1" in param.name or "linear_2" in param.name:
          depth = int(param.name.split("_")[2]) + 2
      else:
          depth = 0
  
      return decay_rate**(n_layers + 2 - depth)
  
  
  @unittest.skipIf(not core.is_compiled_with_cuda(),
                   "core is not compiled with CUDA")
  class TestAdamWOpLayerwiseLR(TestAdamWOp):
      def setUp(self):
          random.seed(2021)
          np.random.seed(2021)
          paddle.seed(2021)
  
      def test_adamw_op_dygraph(self):
          paddle.disable_static()
          value = np.arange(26).reshape(2, 13).astype("float32")
          a = paddle.to_tensor(value)
          linear1 = paddle.nn.Linear(13, 8)
          linear2 = paddle.nn.Linear(8, 5)
  
          # fix the linear name, simple_lr_setting function will use the name
          linear1.weight.name = "linear_1.w_0"
          linear1.bias.name = "linear_1.b_0"
          linear2.weight.name = "linear_2.w_0"
          linear2.bias.name = "linear_2.b_0"
  
          simple_lr_fun = partial(simple_lr_setting, decay_rate=0.8, n_layers=2)
  
          adam = paddle.optimizer.AdamW(
              learning_rate=0.01,
              parameters=[{
                  'params': linear1.parameters()
              }, {
                  'params': linear2.parameters(),
              }],
              apply_decay_param_fun=lambda name: True,
              weight_decay=0.01,
              lr_ratio=simple_lr_fun)
  
          loss_ref = np.array(
              [-1.7267396, -2.81524, -3.9250019, -5.05954, -6.2272625])
          for i in range(5):
              a1 = linear1(a)
              out = linear2(a1)
              out = paddle.mean(out)
              out.backward()
              adam.step()
              adam.clear_gradients()
              np.testing.assert_allclose(out[0].numpy(), loss_ref[i], rtol=1e-6)
  
      def test_adamw_op(self):
          paddle.enable_static()
          place = fluid.CUDAPlace(0)
          train_prog = fluid.Program()
          startup = fluid.Program()
          with fluid.program_guard(train_prog, startup):
              with fluid.unique_name.guard():
                  x = fluid.data(name='x', shape=[None, 10], dtype='float32')
                  y = fluid.data(name='y', shape=[None, 1], dtype='float32')
  
                  fc1 = fluid.layers.fc(input=x, size=32, act=None)
                  prediction = fluid.layers.fc(input=fc1, size=1, act=None)
                  cost = fluid.layers.square_error_cost(input=prediction, label=y)
                  avg_cost = fluid.layers.mean(cost)
  
                  simple_lr_fun = partial(
                      simple_lr_setting, decay_rate=0.8, n_layers=2)
  
                  beta1 = fluid.layers.create_global_var(
                      shape=[1], value=0.85, dtype='float32', persistable=True)
                  beta2 = fluid.layers.create_global_var(
                      shape=[1], value=0.95, dtype='float32', persistable=True)
                  betas = [beta1, beta2]
                  opt = paddle.optimizer.AdamW(
                      learning_rate=1e-5,
                      beta1=beta1,
                      beta2=beta2,
                      weight_decay=0.01,
                      epsilon=1e-8,
                      lr_ratio=simple_lr_fun)
                  opt.minimize(avg_cost)
  
          exe = fluid.Executor(place)
          exe.run(startup)
  
          loss_ref = np.array(
              [0.33895183, 0.3159437, 0.19472016, 0.17764759, 0.1520702])
          for i in range(5):
              inputs = np.random.random(size=[8, 10]).astype('float32')
              outputs = np.random.random(size=[8, 1]).astype('float32')
              rets = exe.run(train_prog,
                             feed={"x": inputs,
                                   "y": outputs},
                             fetch_list=[avg_cost])
              assert rets[0] is not None
              np.testing.assert_allclose(rets[0], loss_ref[i], rtol=1e-6)
  
          paddle.disable_static()
  
  
  if __name__ == "__main__":
      unittest.main()
  