class_docstr: A basic Adam optimizer that includes "correct" L2 weight decay.
  Copied from the google BERT repo.
  Except that in `apply_gradient` function, we add the support to increment
  the passed global step parameter, to make it more compatible to
  tf.train.Optimizer implementation.
class_docstr_tok: ["A","basic","Adam","optimizer","that","includes","``","correct","''","L2","weight","decay",".","Copied","from","the","google","BERT","repo",".","Except","that","in","`","apply_gradient","`","function",",","we","add","the","support","to","increment","the","passed","global","step","parameter",",","to","make","it","more","compatible","to","tf.train.Optimizer","implementation","."]
docstr: See base class.
docstr_tok: ["See","base","class","."]
code: def apply_gradients(
      self, grads_and_vars, global_step=None, name=None
  ):
      """See base class."""
      with tf.name_scope(name, self._name) as name:
          assignments = []
          for grad, param in grads_and_vars:
              if grad is None or param is None:
                  continue
  
              param_name = self._get_variable_name(param.name)
  
              m = tf.get_variable(
                  name=param_name + "/adam_m",
                  shape=param.shape.as_list(),
                  dtype=tf.float32,
                  trainable=False,
                  initializer=tf.zeros_initializer(),
              )
              v = tf.get_variable(
                  name=param_name + "/adam_v",
                  shape=param.shape.as_list(),
                  dtype=tf.float32,
                  trainable=False,
                  initializer=tf.zeros_initializer(),
              )
  
              # Standard Adam update.
              next_m = tf.multiply(self.beta_1, m) + tf.multiply(
                  1.0 - self.beta_1, grad
              )
              next_v = tf.multiply(self.beta_2, v) + tf.multiply(
                  1.0 - self.beta_2, tf.square(grad)
              )
  
              update = next_m / (tf.sqrt(next_v) + self.epsilon)
  
              # Just adding the square of the weights to the loss function is
              # *not* the correct way of using L2 regularization/weight decay
              # with Adam, since that will interact with the m and v
              # parameters in strange ways.
              # Instead we want ot decay the weights in a manner that doesn't
              # interact with the m/v parameters.
              # This is equivalent to adding the square
              # of the weights to the loss with plain (non-momentum) SGD.
              if self._do_use_weight_decay(param_name):
                  update += self.weight_decay_rate * param
  
              update_with_lr = self.learning_rate * update
  
              next_param = param - update_with_lr
  
              assignments.extend(
                  [
                      param.assign(next_param),
                      m.assign(next_m),
                      v.assign(next_v),
                  ]
              )
  
          update_ops = assignments
          if global_step is None:
              apply_updates = self._finish(update_ops, name)
          else:
              with tf.control_dependencies(
                  [self._finish(update_ops, "update")]
              ):
                  with tf.colocate_with(global_step):
                      apply_updates = tf.assign_add(
                          global_step, 1, name=name
                      )
  
      return apply_updates
  
file_path: texar/texar/tf/core/optimization.py
imports: import re
  import tensorflow
  from texar.tf.hyperparams import HParams
  from texar.tf.utils import utils
label_desc: Use image node_id 2 for calling the AdamWeightDecayOptimizer obj's underlying member method code with example usage: obj.apply_gradients(grads_and_vars, global_step, name) and returns: apply_updates