import tensorflow as tf

            self._zeros_slot(v, "g", self._name)

    def _apply_dense(self, grad, var):
        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
        beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
        beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
        if var.dtype.base_dtype == tf.float16:
            eps = 1e-7  # Can't use 1e-8 due to underflow -- not sure if it makes a big difference.
        else:
            eps = 1e-8

        v = self.get_slot(var, "v")
        v_t = v.assign(beta2_t * v + (1. - beta2_t) * tf.square(grad))
        m = self.get_slot(var, "m")
        m_t = m.assign(beta1_t * m + (1. - beta1_t) * grad)
        v_t_hat = tf.div(v_t, 1. - beta2_t)
        m_t_hat = tf.div(m_t, 1. - beta1_t)

        g_t = tf.div(m_t_hat, tf.sqrt(v_t_hat) + eps)
        g_t_1 = self.get_slot(var, "g")
        g_t = g_t_1.assign(g_t)

        var_update = state_ops.assign_sub(var,
                                          2. * lr_t * g_t - lr_t * g_t_1)  # Adam would be lr_t * g_t
        return control_flow_ops.group(*[var_update, m_t, v_t, g_t])

    def _apply_sparse(self, grad, var):
        raise NotImplementedError("Sparse gradient updates are not supported.")

class RegularizeGradientDescentOptimizer(optimizer.Optimizer):
