class_docstr: Update rule of Adam optimization algorithm.
  
  See: `Adam: A Method for Stochastic Optimization
  <https://arxiv.org/abs/1412.6980v8>`_
  
  Modified for proper weight decay.
  
  See: `Fixing Weight Decay Regularization in Adam
  <https://openreview.net/forum?id=rk6qdGgCZ>`_
  
  With option to use AMSGrad variant of Adam.
  
  See: `On the Convergence of Adam and Beyond
  <https://openreview.net/forum?id=ryQu7f-RZ>`_
  
  With option to use AdaBound variant of Adam.
  
  See: `Adaptive Gradient Methods with Dynamic Bound of Learning Rate
  <https://openreview.net/forum?id=Bkg3g2R9FX>`
  
  See :class:`~chainer.optimizers.Adam` for the default values
  of the hyperparameters.
  
  Args:
      parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter
          that provides the default values.
      alpha (float): Coefficient of learning rate.
      beta1 (float): Exponential decay rate of the first order moment.
      beta2 (float): Exponential decay rate of the second order moment.
      eps (float): Small value for the numerical stability.
      eta (float): Schedule multiplier, can be used for warm restarts.
      weight_decay_rate (float): Weight decay rate.
      amsgrad (bool): Whether to use the AMSGrad variant of Adam.
      adabound (bool): Whether to use the AdaBound variant of Adam.
      final_lr (float): Final (SGD) learning rate in AdaBound.
      gamma (float): Convergence speed of the bound functions in AdaBound.
class_docstr_tok: ["Update","rule","of","Adam","optimization","algorithm",".","See",":","`","Adam",":","A","Method","for","Stochastic","Optimization","<","https",":","\/\/arxiv.org\/abs\/1412.6980v8",">","`","_","Modified","for","proper","weight","decay",".","See",":","`","Fixing","Weight","Decay","Regularization","in","Adam","<","https",":","\/\/openreview.net\/forum","?","id=rk6qdGgCZ",">","`","_","With","option","to","use","AMSGrad","variant","of","Adam",".","See",":","`","On","the","Convergence","of","Adam","and","Beyond","<","https",":","\/\/openreview.net\/forum","?","id=ryQu7f-RZ",">","`","_","With","option","to","use","AdaBound","variant","of","Adam",".","See",":","`","Adaptive","Gradient","Methods","with","Dynamic","Bound","of","Learning","Rate","<","https",":","\/\/openreview.net\/forum","?","id=Bkg3g2R9FX",">","`","See",":","class",":","`","~chainer.optimizers.Adam","`","for","the","default","values","of","the","hyperparameters",".","Args",":","parent_hyperparam","(","~chainer.optimizer.Hyperparameter",")",":","Hyperparameter","that","provides","the","default","values",".","alpha","(","float",")",":","Coefficient","of","learning","rate",".","beta1","(","float",")",":","Exponential","decay","rate","of","the","first","order","moment",".","beta2","(","float",")",":","Exponential","decay","rate","of","the","second","order","moment",".","eps","(","float",")",":","Small","value","for","the","numerical","stability",".","eta","(","float",")",":","Schedule","multiplier",",","can","be","used","for","warm","restarts",".","weight_decay_rate","(","float",")",":","Weight","decay","rate",".","amsgrad","(","bool",")",":","Whether","to","use","the","AMSGrad","variant","of","Adam",".","adabound","(","bool",")",":","Whether","to","use","the","AdaBound","variant","of","Adam",".","final_lr","(","float",")",":","Final","(","SGD",")","learning","rate","in","AdaBound",".","gamma","(","float",")",":","Convergence","speed","of","the","bound","functions","in","AdaBound","."]
code: def update_core_gpu(self, param):
      grad = param.grad
      if grad is None:
          return
      hp = self.hyperparam
      dtype = _get_intermediate_dtype(param.dtype.type)
      self._check_eps(dtype)
  
      if self._dummy is None:
          self._dummy = cuda.cupy.empty((0,), dtype=dtype)
  
      if hp.adabound:
          lower, upper = self.bounds
      if hp.amsgrad and hp.adabound:
          if AdamRule._amsbound_kernel is None:
              AdamRule._amsbound_kernel = cuda.elementwise(
                  "P grad, T alpha_t, T one_minus_beta1, T one_minus_beta2, "
                  "T lower, T upper, "
                  "T eps, T eta, T weight_decay_rate, raw T dummy",
                  "P param, P m, P v, P vhat",
                  """T grad_ = static_cast<T>(grad);
                         T m_ = static_cast<T>(m);
                         T v_ = static_cast<T>(v);
                         T vhat_ = static_cast<T>(vhat);
                         m_ += one_minus_beta1 * (grad_ - m_);
                         v_ += one_minus_beta2 * (grad_ * grad_ - v_);
                         vhat_ = max(vhat_, v_);
                         vhat = static_cast<T>(vhat_);
                         m = static_cast<P>(m_);
                         v = static_cast<P>(v_);
                         param -= eta *
                             (max(min(alpha_t / (sqrt(vhat_) + eps), upper),
                                  lower) * m_ + weight_decay_rate * param);""",
                  "amsbound",
              )
          AdamRule._amsbound_kernel(
              grad,
              self.alpha_t,
              1 - hp.beta1,
              1 - hp.beta2,
              lower,
              upper,
              hp.eps,
              hp.eta,
              hp.weight_decay_rate,
              self._dummy,
              param.data,
              self.state["m"],
              self.state["v"],
              self.state["vhat"],
          )
      elif hp.adabound:
          if AdamRule._adabound_kernel is None:
              AdamRule._adabound_kernel = cuda.elementwise(
                  "P grad, T alpha_t, T one_minus_beta1, T one_minus_beta2, "
                  "T lower, T upper, "
                  "T eps, T eta, T weight_decay_rate, raw T dummy",
                  "P param, P m, P v",
                  """T grad_ = static_cast<T>(grad);
                         T m_ = static_cast<T>(m);
                         T v_ = static_cast<T>(v);
                         m_ += one_minus_beta1 * (grad_ - m_);
                         v_ += one_minus_beta2 * (grad_ * grad_ - v_);
                         m = static_cast<P>(m_);
                         v = static_cast<P>(v_);
                         param -= eta *
                             (max(min(alpha_t / (sqrt(v_) + eps), upper),
                                  lower) * m_ + weight_decay_rate * param);""",
                  "adabound",
              )
          AdamRule._adabound_kernel(
              grad,
              self.alpha_t,
              1 - hp.beta1,
              1 - hp.beta2,
              lower,
              upper,
              hp.eps,
              hp.eta,
              hp.weight_decay_rate,
              self._dummy,
              param.data,
              self.state["m"],
              self.state["v"],
          )
      elif hp.amsgrad:
          if AdamRule._amsgrad_kernel is None:
              AdamRule._amsgrad_kernel = cuda.elementwise(
                  "P grad, T alpha_t, T one_minus_beta1, T one_minus_beta2, "
                  "T eps, T eta, T weight_decay_rate, raw T dummy",
                  "P param, P m, P v, P vhat",
                  """T grad_ = static_cast<T>(grad);
                         T m_ = static_cast<T>(m);
                         T v_ = static_cast<T>(v);
                         T vhat_ = static_cast<T>(vhat);
                         m_ += one_minus_beta1 * (grad_ - m_);
                         v_ += one_minus_beta2 * (grad_ * grad_ - v_);
                         vhat_ = max(vhat_, v_);
                         vhat = static_cast<T>(vhat_);
                         m = static_cast<P>(m_);
                         v = static_cast<P>(v_);
                         param -= eta * (alpha_t * m_ / (sqrt(vhat_) + eps) +
                                         weight_decay_rate * param);""",
                  "adam",
              )
          AdamRule._amsgrad_kernel(
              grad,
              self.alpha_t,
              1 - hp.beta1,
              1 - hp.beta2,
              hp.eps,
              hp.eta,
              hp.weight_decay_rate,
              self._dummy,
              param.data,
              self.state["m"],
              self.state["v"],
              self.state["vhat"],
          )
      else:
          if AdamRule._kernel is None:
              AdamRule._kernel = cuda.elementwise(
                  "P grad, T alpha_t, T one_minus_beta1, T one_minus_beta2, "
                  "T eps, T eta, T weight_decay_rate, raw T dummy",
                  "P param, P m, P v",
                  """T grad_ = static_cast<T>(grad);
                         T m_ = static_cast<T>(m);
                         T v_ = static_cast<T>(v);
                         m_ += one_minus_beta1 * (grad_ - m_);
                         v_ += one_minus_beta2 * (grad_ * grad_ - v_);
                         m = static_cast<P>(m_);
                         v = static_cast<P>(v_);
                         param -= eta * (alpha_t * m_ / (sqrt(v_) + eps) +
                                         weight_decay_rate * param);""",
                  "adam",
              )
          AdamRule._kernel(
              grad,
              self.alpha_t,
              1 - hp.beta1,
              1 - hp.beta2,
              hp.eps,
              hp.eta,
              hp.weight_decay_rate,
              self._dummy,
              param.data,
              self.state["m"],
              self.state["v"],
          )
  
imports: from __future__ import division
  import math
  import warnings
  import numpy
  import chainer
  from chainer.backends import cuda
  from chainer.backends import intel64
  from chainer import optimizer
  from chainer import types
label_desc: Use image node_id 5 for calling the AdamRule obj's underlying member method code with example usage: obj.update_core_gpu(param) without return types