      def upscale(self, input_tensor, filters, kernel_size=3, padding="same",
                  use_instance_norm=False, res_block_follows=False, scale_factor=2, **kwargs):
          """ An upscale layer for sub-pixel up-scaling.
  
          Adds reflection padding if it has been selected by the user, and other post-processing
          if requested by the plugin.
  
          Parameters
          ----------
          input_tensor: tensor
              The input tensor to the layer
 
</s> remove         logger.debug("input_tensor: %s, filters: %s, kernel_size: %s, strides: %s, "
                     "use_instance_norm: %s, kwargs: %s)", input_tensor, filters, kernel_size,
                     strides, use_instance_norm, kwargs)
        name = self._get_name("conv_{}".format(input_tensor.shape[1]))
        if self.use_reflect_padding:
            input_tensor = ReflectionPadding2D(
                stride=strides,
                kernel_size=kernel_size,
                name="{}_reflectionpadding2d".format(name))(input_tensor)
            padding = "valid"
        var_x = self.conv2d(input_tensor, filters,
                            kernel_size=kernel_size,
                            strides=strides,
                            padding=padding,
                            name="{}_conv2d".format(name),
                            **kwargs)
        if use_instance_norm:
            var_x = InstanceNormalization(name="{}_instancenorm".format(name))(var_x)
        if not res_block_follows:
            var_x = LeakyReLU(0.1, name="{}_leakyrelu".format(name))(var_x)
        return var_x

    def upscale(self, input_tensor, filters, kernel_size=3, padding="same",
                use_instance_norm=False, res_block_follows=False, scale_factor=2, **kwargs):
        """ An upscale layer for sub-pixel up-scaling.
 </s> add         if initializer is None:
            retval = ConvolutionAware() if _CONFIG["conv_aware_init"] else he_uniform()
            logger.debug("Set default kernel_initializer: %s", retval)
        else:
            retval = initializer
            logger.debug("Using model supplied initializer: %s", retval)
        return retval </s> remove     # <<< Original Model Blocks >>> #
    def conv(self, input_tensor, filters, kernel_size=5, strides=2, padding="same",
             use_instance_norm=False, res_block_follows=False, **kwargs):
        """ A standard Convolution 2D layer which applies user specified configuration to the
        layer.
 </s> add  </s> remove         Adds reflection padding if it has been selected by the user, and other post-processing
        if requested by the plugin.
 </s> add     def __init__(self,
                 filters,
                 kernel_size=3,
                 padding="same",
                 scale_factor=2,
                 use_instance_norm=False,
                 res_block_follows=False,
                 **kwargs):
        self._name = _get_name("upscale_{}".format(filters))
        logger.debug("name: %s. filters: %s, kernel_size: %s, padding: %s, scale_factor: %s, "
                     "use_instance_norm: %s, res_block_follows: %s, kwargs: %s)",
                     self._name, filters, kernel_size, padding, scale_factor, use_instance_norm,
                     res_block_follows, kwargs)

        self._filters = filters
        self._kernel_size = kernel_size
        self._padding = padding
        self._scale_factor = scale_factor
        self._use_instance_norm = use_instance_norm
        self._res_block_follows = res_block_follows
        self._kwargs = kwargs

    def __call__(self, inputs):
        """ Call the Faceswap Convolutional Layer. </s> remove         Most of up-scaling is approximating lighting gradients which can be accurately achieved
        using linear fitting. This layer attempts to improve memory consumption by splitting
        with bilinear and convolutional layers so that the sub-pixel update will get details
        whilst the bilinear filter will get lighting.
 </s> add #  << UPSCALING >>

class UpscaleBlock():  # pylint:disable=too-few-public-methods
    """ An upscale layer for sub-pixel up-scaling.

    Adds reflection padding if it has been selected by the user, and other post-processing
    if requested by the plugin.

    Parameters
    ----------
    filters: int
        The dimensionality of the output space (i.e. the number of output filters in the
        convolution)
    kernel_size: int, optional
        An integer or tuple/list of 2 integers, specifying the height and width of the 2D
        convolution window. Can be a single integer to specify the same value for all spatial
        dimensions. Default: 3
    padding: ["valid", "same"], optional
        The padding to use. NB: If reflect padding has been selected in the user configuration
        options, then this argument will be ignored in favor of reflect padding. Default: `"same"`
    scale_factor: int, optional
        The amount to upscale the image. Default: `2`
    use_instance_norm: bool, optional
        ``True`` if instance normalization should be applied after the convolutional layer.
        Default: ``False``
    res_block_follows: bool, optional
        If a residual block will follow this layer, then this should be set to ``True`` to add
        a leaky ReLu after the convolutional layer. Default: ``False``
    kwargs: dict
        Any additional Keras standard layer keyword arguments to pass to the Convolutional 2D layer
    """ </s> remove         original = kwargs.get("kernel_initializer", None)
        kwargs["kernel_initializer"] = initializer
        logger.debug("Switched kernel_initializer from %s to %s", original, initializer)
        return original
 </s> add         var_x = Conv2D(self._filters,
                       self._kernel_size,
                       padding=self._padding,
                       name="{}_conv2d".format(self._name),
                       **self._kwargs)(inputs)
        var_x = Activation(self._activation, dtype="float32", name=self._name)(var_x)
        return var_x


class Conv2DBlock():  # pylint:disable=too-few-public-methods
    """ A standard Convolution 2D layer which applies user specified configuration to the
    layer.

    Adds reflection padding if it has been selected by the user, and other post-processing
    if requested by the plugin.
    def __call__(self, inputs):
        """ Call the Faceswap Separable Convolutional 2D Block.
      def upscale(self, input_tensor, filters, kernel_size=3, padding="same",
                  use_instance_norm=False, res_block_follows=False, scale_factor=2, **kwargs):
          """ An upscale layer for sub-pixel up-scaling.
  
    def __call__(self, inputs):
        """ Call the Faceswap Separable Convolutional 2D Block.
    def __call__(self, inputs):
        """ Call the Faceswap Separable Convolutional 2D Block.
  
          Parameters
          ----------
          input_tensor: tensor
              The input tensor to the layer
keep keep keep keep replace replace keep keep keep keep keep
 Faceswap 2.0 (#1045)

* Core Updates
    - Remove lib.utils.keras_backend_quiet and replace with get_backend() where relevant
    - Document lib.gpu_stats and lib.sys_info
    - Remove call to GPUStats.is_plaidml from convert and replace with get_backend()
    - lib.gui.menu - typofix

* Update Dependencies
Bump Tensorflow Version Check

* Port extraction to tf2

* Add custom import finder for loading Keras or tf.keras depending on backend

* Add `tensorflow` to KerasFinder search path

* Basic TF2 training running

* model.initializers - docstring fix

* Fix and pass tests for tf2

* Replace Keras backend tests with faceswap backend tests

* Initial optimizers update

* Monkey patch tf.keras optimizer

* Remove custom Adam Optimizers and Memory Saving Gradients

* Remove multi-gpu option. Add Distribution to cli

* plugins.train.model._base: Add Mirror, Central and Default distribution strategies

* Update tensorboard kwargs for tf2

* Penalized Loss - Fix for TF2 and AMD

* Fix syntax for tf2.1

* requirements typo fix

* Explicit None for clipnorm if using a distribution strategy

* Fix penalized loss for distribution strategies

* Update Dlight

* typo fix

* Pin to TF2.2

* setup.py - Install tensorflow from pip if not available in Conda

* Add reduction options and set default for mirrored distribution strategy

* Explicitly use default strategy rather than nullcontext

* lib.model.backup_restore documentation

* Remove mirrored strategy reduction method and default based on OS

* Initial restructure - training

* Remove PingPong
Start model.base refactor

* Model saving and resuming enabled

* More tidying up of model.base

* Enable backup and snapshotting

* Re-enable state file
Remove loss names from state file
Fix print loss function
Set snapshot iterations correctly

* Revert original model to Keras Model structure rather than custom layer
Output full model and sub model summary
Change NNBlocks to callables rather than custom keras layers

* Apply custom Conv2D layer

* Finalize NNBlock restructure
Update Dfaker blocks

* Fix reloading model under a different distribution strategy

* Pass command line arguments through to trainer

* Remove training_opts from model and reference params directly

* Tidy up model __init__

* Re-enable tensorboard logging
Suppress "Model Not Compiled" warning

* Fix timelapse

* lib.model.nnblocks - Bugfix residual block
Port dfaker
bugfix original

* dfl-h128 ported

* DFL SAE ported

* IAE Ported

* dlight ported

* port lightweight

* realface ported

* unbalanced ported

* villain ported

* lib.cli.args - Update Batchsize + move allow_growth to config

* Remove output shape definition
Get image sizes per side rather than globally

* Strip mask input from encoder

* Fix learn mask and output learned mask to preview

* Trigger Allow Growth prior to setting strategy

* Fix GUI Graphing

* GUI - Display batchsize correctly + fix training graphs

* Fix penalized loss

* Enable mixed precision training

* Update analysis displayed batch to match input

* Penalized Loss - Multi-GPU Fix

* Fix all losses for TF2

* Fix Reflect Padding

* Allow different input size for each side of the model

* Fix conv-aware initialization on reload

* Switch allow_growth order

* Move mixed_precision to cli

* Remove distrubution strategies

* Compile penalized loss sub-function into LossContainer

* Bump default save interval to 250
Generate preview on first iteration but don't save
Fix iterations to start at 1 instead of 0
Remove training deprecation warnings
Bump some scripts.train loglevels

* Add ability to refresh preview on demand on pop-up window

* Enable refresh of training preview from GUI

* Fix Convert
Debug logging in Initializers

* Fix Preview Tool

* Update Legacy TF1 weights to TF2
Catch stats error on loading stats with missing logs

* lib.gui.popup_configure - Make more responsive + document

* Multiple Outputs supported in trainer
Original Model - Mask output bugfix

* Make universal inference model for convert
Remove scaling from penalized mask loss (now handled at input to y_true)

* Fix inference model to work properly with all models

* Fix multi-scale output for convert

* Fix clipnorm issue with distribution strategies
Edit error message on OOM

* Update plaidml losses

* Add missing file

* Disable gmsd loss for plaidnl

* PlaidML - Basic training working

* clipnorm rewriting for mixed-precision

* Inference model creation bugfixes

* Remove debug code

* Bugfix: Default clipnorm to 1.0

* Remove all mask inputs from training code

* Remove mask inputs from convert

* GUI - Analysis Tab - Docstrings

* Fix rate in totals row

* lib.gui - Only update display pages if they have focus

* Save the model on first iteration

* plaidml - Fix SSIM loss with penalized loss

* tools.alignments - Remove manual and fix jobs

* GUI - Remove case formatting on help text

* gui MultiSelect custom widget - Set default values on init

* vgg_face2 - Move to plugins.extract.recognition and use plugins._base base class
cli - Add global GPU Exclude Option
tools.sort - Use global GPU Exlude option for backend
lib.model.session - Exclude all GPUs when running in CPU mode
lib.cli.launcher - Set backend to CPU mode when all GPUs excluded

* Cascade excluded devices to GPU Stats

* Explicit GPU selection for Train and Convert

* Reduce Tensorflow Min GPU Multiprocessor Count to 4

* remove compat.v1 code from extract

* Force TF to skip mixed precision compatibility check if GPUs have been filtered

* Add notes to config for non-working AMD losses

* Rasie error if forcing extract to CPU mode

* Fix loading of legace dfl-sae weights + dfl-sae typo fix

* Remove unused requirements
Update sphinx requirements
Fix broken rst file locations

* docs: lib.gui.display

* clipnorm amd condition check

* documentation - gui.display_analysis

* Documentation - gui.popup_configure

* Documentation - lib.logger

* Documentation - lib.model.initializers

* Documentation - lib.model.layers

* Documentation - lib.model.losses

* Documentation - lib.model.nn_blocks

* Documetation - lib.model.normalization

* Documentation - lib.model.session

* Documentation - lib.plaidml_stats

* Documentation: lib.training_data

* Documentation: lib.utils

* Documentation: plugins.train.model._base

* GUI Stats: prevent stats from using GPU

* Documentation - Original Model

* Documentation: plugins.model.trainer._base

* linting

* unit tests: initializers + losses

* unit tests: nn_blocks

* bugfix - Exclude gpu devices in train, not include

* Enable Exclude-Gpus in Extract

* Enable exclude gpus in tools

* Disallow multiple plugin types in a single model folder

* Automatically add exclude_gpus argument in for cpu backends

* Cpu backend fixes

* Relax optimizer test threshold

* Default Train settings - Set mask to Extended

* Update Extractor cli help text
Update to Python 3.8

* Fix FAN to run on CPU

* lib.plaidml_tools - typofix

* Linux installer - check for curl

* linux installer - typo fix 
lib/model/nn_blocks.py