  
      def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
                   epsilon=None, decay=0., amsgrad=False, cpu_mode=0, **kwargs):
          super().__init__(lr, beta_1, beta_2, epsilon, decay, **kwargs)
          self.cpu_mode = self.set_cpu_mode(cpu_mode)
  
      @staticmethod
      def set_cpu_mode(cpu_mode):
          """ Set the CPU mode to 0 if not using tensorflow, else passed in arg """
          retval = False if K.backend() != "tensorflow" else cpu_mode
 
</s> remove     def set_cpu_mode(cpu_mode):
        """ Set the CPU mode to 0 if not using tensorflow, else passed in arg """
 </s> add     def _set_cpu_mode(cpu_mode):
        """ Sets the CPU mode to False if not using Tensorflow, otherwise the given value.

        Parameters
        ----------
        cpu_mode: bool
            Set to ``True`` to perform some of the calculations on CPU for Nvidia backends,
            otherwise ``False``.

        Returns
        -------
        bool
            ``True`` if some calculations should be performed on CPU otherwise ``False``
        """ </s> remove     def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
                 epsilon=None, decay=0., amsgrad=False, cpu_mode=0, **kwargs):
 </s> add     def __init__(self,
                 lr=0.001,
                 beta_1=0.9,
                 beta_2=0.999,
                 epsilon=None,
                 decay=0.,
                 amsgrad=False,
                 cpu_mode=False,
                 **kwargs): </s> remove     """Adapted Keras Adam Optimizer to allow support of calculations
       on CPU for Tensorflow.

       Adapted from https://github.com/iperov/DeepFaceLab
 </s> add     """Adapted Keras Adam Optimizer to allow support of calculations on CPU for Tensorflow.

    Default parameters follow those provided in the original paper. Adapted from
    https://github.com/iperov/DeepFaceLab

    Parameters
    ----------
    lr: float, optional
        >= `0`. Learning rate. Default: `0.001`
    beta_1: float, optional
        `0` < beta < `1` Generally close to `1`. Default: `0.9`
    beta_2: float, optional
        `0` < beta < `1`. Generally close to `1`. Default: `0.999`
    epsilon: float, optional
        >= `0`. Fuzz factor. If ``None``, defaults to `K.epsilon()`. Default: ``None``
    decay: float, optional
        >= 0. Learning rate decay over each update. Default: `0`
    amsgrad: bool, optional
        ``True`` to apply the AMSGrad variant of this algorithm from the paper "On the Convergence
        of Adam and Beyond" otherwise ``False``. Default: ``False``
    cpu_mode: bool, optional
        Set to ``True`` to perform some of the calculations on CPU for Nvidia backends, otherwise
        ``False``. Default: ``False``
    kwargs: dict
        Any additional standard Keras optimizer keyword arguments

    References
    ----------
    - Adam - A Method for Stochastic Optimization - https://arxiv.org/abs/1412.6980v8

    - On the Convergence of Adam and Beyond - https://openreview.net/forum?id=ryQu7f-RZ </s> remove     def update_1(self, params):
        """ First update on CPU or GPU """
 </s> add     def _update_1(self, params):
        """ Perform the first update. Run under CPU context if running on Tensorflow and CPU mode
        is enabled, otherwise run on the default device. """ </s> remove 
# <<< GAN V2.2 Blocks >>> #
# TODO Merge these into NNBLock class when porting GAN2.2


# Gan Constansts:
GAN22_CONV_INIT = "he_normal"
GAN22_REGULARIZER = 1e-4


# Gan Blocks:
def normalization(inp, norm="none", group="16"):
    """ GAN Normalization """
    if norm == "layernorm":
        var_x = GroupNormalization(group=group)(inp)
    elif norm == "batchnorm":
        var_x = BatchNormalization()(inp)
    elif norm == "groupnorm":
        var_x = GroupNormalization(group=16)(inp)
    elif norm == "instancenorm":
        var_x = InstanceNormalization()(inp)
    elif norm == "hybrid":
        if group % 2 == 1:
            raise ValueError("Output channels must be an even number for hybrid norm, "
                             "received {}.".format(group))
        filt = group
        var_x_0 = Lambda(lambda var_x: var_x[..., :filt // 2])(var_x)
        var_x_1 = Lambda(lambda var_x: var_x[..., filt // 2:])(var_x)
        var_x_0 = Conv2D(filt // 2,
                         kernel_size=1,
                         kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                         kernel_initializer=GAN22_CONV_INIT)(var_x_0)
        var_x_1 = InstanceNormalization()(var_x_1)
        var_x = concatenate([var_x_0, var_x_1], axis=-1)
    else:
        var_x = inp
    return var_x


def upscale_ps(inp, filters, initializer, use_norm=False, norm="none"):
    """ GAN Upscaler - Pixel Shuffler """
    var_x = Conv2D(filters * 4,
                   kernel_size=3,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer=initializer,
                   padding="same")(inp)
    var_x = LeakyReLU(0.2)(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    var_x = PixelShuffler()(var_x)
    return var_x


def upscale_nn(inp, filters, use_norm=False, norm="none"):
    """ GAN Neural Network """
    var_x = UpSampling2D()(inp)
    var_x = reflect_padding_2d(var_x, 1)
    var_x = Conv2D(filters,
                   kernel_size=3,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer="he_normal")(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    return var_x


def reflect_padding_2d(inp, pad=1):
    """ GAN Reflect Padding (2D) """
    var_x = Lambda(lambda var_x: tf.pad(var_x,
                                        [[0, 0], [pad, pad], [pad, pad], [0, 0]],
                                        mode="REFLECT"))(inp)
    return var_x


def conv_gan(inp, filters, use_norm=False, strides=2, norm="none"):
    """ GAN Conv Block """
    var_x = Conv2D(filters,
                   kernel_size=3,
                   strides=strides,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer=GAN22_CONV_INIT,
                   use_bias=False,
                   padding="same")(inp)
    var_x = Activation("relu")(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    return var_x


def conv_d_gan(inp, filters, use_norm=False, norm="none"):
    """ GAN Discriminator Conv Block """
    var_x = inp
    var_x = Conv2D(filters,
                   kernel_size=4,
                   strides=2,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer=GAN22_CONV_INIT,
                   use_bias=False,
                   padding="same")(var_x)
    var_x = LeakyReLU(alpha=0.2)(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    return var_x


def res_block_gan(inp, filters, use_norm=False, norm="none"):
    """ GAN Res Block """
    var_x = Conv2D(filters,
                   kernel_size=3,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer=GAN22_CONV_INIT,
                   use_bias=False,
                   padding="same")(inp)
    var_x = LeakyReLU(alpha=0.2)(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    var_x = Conv2D(filters,
                   kernel_size=3,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER),
                   kernel_initializer=GAN22_CONV_INIT,
                   use_bias=False,
                   padding="same")(var_x)
    var_x = add([var_x, inp])
    var_x = LeakyReLU(alpha=0.2)(var_x)
    var_x = normalization(var_x, norm, filters) if use_norm else var_x
    return var_x


def self_attn_block(inp, n_c, squeeze_factor=8):
    """ GAN Self Attention Block
    Code borrows from https://github.com/taki0112/Self-Attention-GAN-Tensorflow
    """
    msg = "Input channels must be >= {}, recieved nc={}".format(squeeze_factor, n_c)
    assert n_c // squeeze_factor > 0, msg
    var_x = inp
    shape_x = var_x.get_shape().as_list()

    var_f = Conv2D(n_c // squeeze_factor, 1,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)
    var_g = Conv2D(n_c // squeeze_factor, 1,
                   kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)
    var_h = Conv2D(n_c, 1, kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)

    shape_f = var_f.get_shape().as_list()
    shape_g = var_g.get_shape().as_list()
    shape_h = var_h.get_shape().as_list()
    flat_f = Reshape((-1, shape_f[-1]))(var_f)
    flat_g = Reshape((-1, shape_g[-1]))(var_g)
    flat_h = Reshape((-1, shape_h[-1]))(var_h)

    var_s = Lambda(lambda var_x: K.batch_dot(var_x[0],
                                             Permute((2, 1))(var_x[1])))([flat_g, flat_f])

    beta = Softmax(axis=-1)(var_s)
    var_o = Lambda(lambda var_x: K.batch_dot(var_x[0], var_x[1]))([beta, flat_h])
    var_o = Reshape(shape_x[1:])(var_o)
    var_o = Scale()(var_o)

    out = add([var_o, inp])
    return out
 </s> add 
        self.cpu_mode = self._set_cpu_mode(cpu_mode)
  
      def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
                   epsilon=None, decay=0., amsgrad=False, cpu_mode=0, **kwargs):
          super().__init__(lr, beta_1, beta_2, epsilon, decay, **kwargs)
        self.cpu_mode = self._set_cpu_mode(cpu_mode)
  
      @staticmethod
      def set_cpu_mode(cpu_mode):
          """ Set the CPU mode to 0 if not using tensorflow, else passed in arg """
          retval = False if K.backend() != "tensorflow" else cpu_mode
keep keep keep keep replace keep keep keep keep keep
 Simple backend unit tests (#1020)

* Add simple backend tests for lib.model
* Document lib.model
* Fix GMSD Loss for AMD
* Remove obsolete code from lib.model 
lib/model/optimizers.py