          if not self._is_predict:
              self._state.add_session_loss_names(loss.names)
          logger.debug("Compiled Model: %s", self._model)
  
      def _get_optimizer(self):
          """ Return a Keras Adam Optimizer with user selected parameters.
  
          Returns
          -------
          :class:`keras.optimizers.Adam`
              An Adam Optimizer with the given user settings
  
          Notes
          -----
          Clip-norm is ballooning VRAM usage, which is not expected behavior and may be a bug in
          Keras/Tensorflow.
  
          PlaidML has a bug regarding the clip-norm parameter See:
          https://github.com/plaidml/plaidml/issues/228. We workaround by simply not adding this
          parameter for AMD backend users.
          """
          kwargs = dict(beta_1=0.5, beta_2=0.99)
  
          learning_rate = "lr" if get_backend() == "amd" else "learning_rate"
          kwargs[learning_rate] = self.config.get("learning_rate", 5e-5)
  
          clipnorm = self.config.get("clipnorm", False)
          if clipnorm and (self._args.distributed or self._args.mixed_precision):
              logger.warning("Clipnorm has been selected, but is unsupported when using distributed "
                             "or mixed_precision training, so has been disabled. If you wish to "
                             "enable clipnorm, then you must disable these options.")
              clipnorm = False
          if clipnorm and get_backend() == "amd":
              # TODO add clipnorm in for plaidML when it is fixed upstream. Still not fixed in
              # release 0.7.0.
              logger.warning("Due to a bug in plaidML, clipnorm cannot be used on AMD backends so "
                             "has been disabled")
              clipnorm = False
          if clipnorm:
              kwargs["clipnorm"] = 1.0
  
          retval = Adam(**kwargs)
          if self._settings.use_mixed_precision:
              retval = self._settings.LossScaleOptimizer(retval, loss_scale="dynamic")
          logger.debug("Optimizer: %s, kwargs: %s", retval, kwargs)
          return retval
  
      def _legacy_mapping(self):  # pylint:disable=no-self-use
          """ The mapping of separate model files to single model layers for transferring of legacy
          weights.
  
          Returns
 
</s> remove         optimizer = self._get_optimizer()
 </s> add         optimizer = _Optimizer(self.config["optimizer"],
                               self.config["learning_rate"],
                               self.config.get("clipnorm", False),
                               self._args).optimizer
        if self._settings.use_mixed_precision:
            optimizer = self._settings.LossScaleOptimizer(optimizer, loss_scale="dynamic")
 </s> remove             section=section, title="learning_rate", datatype=float, default=5e-5,
            min_max=(1e-6, 1e-4), rounding=6, fixed=False, group="optimizer",
            info="Learning rate - how fast your network will learn (how large are "
                 "the modifications to the model weights after one batch of training). "
                 "Values that are too large might result in model crashes and the "
                 "inability of the model to find the best solution. "
                 "Values that are too small might be unable to escape from dead-ends "
                 "and find the best global minimum.")
 </s> add             section=section,
            title="icnr_init",
            datatype=bool,
            default=False,
            group="initialization",
            info="Use ICNR to tile the default initializer in a repeating pattern. "
                 "This strategy is designed for pairing with sub-pixel / pixel shuffler "
                 "to reduce the 'checkerboard effect' in image reconstruction. "
                 "\n\t https://arxiv.org/ftp/arxiv/papers/1707/1707.02937.pdf")
        self.add_item(
            section=section,
            title="conv_aware_init",
            datatype=bool,
            default=False,
            group="initialization",
            info="Use Convolution Aware Initialization for convolutional layers. "
                 "This can help eradicate the vanishing and exploding gradient problem "
                 "as well as lead to higher accuracy, lower loss and faster convergence.\nNB:"
                 "\n\t This can use more VRAM when creating a new model so you may want to "
                 "lower the batch size for the first run. The batch size can be raised "
                 "again when reloading the model. "
                 "\n\t Multi-GPU is not supported for this option, so you should start the model "
                 "on a single GPU. Once training has started, you can stop training, enable "
                 "multi-GPU and resume."
                 "\n\t Building the model will likely take several minutes as the calculations "
                 "for this initialization technique are expensive. This will only impact starting "
                 "a new model.")
        self.add_item(
            section=section,
            title="optimizer",
            datatype=str,
            gui_radio=True,
            group="optimizer",
            default="adam",
            choices=["adam", "nadam", "rms-prop"],
            info="The optimizer to use."
                 "\n\t adam - Adaptive Moment Optimization. A stochastic gradient descent method "
                 "that is based on adaptive estimation of first-order and second-order moments."
                 "\n\t nadam - Adaptive Moment Optimization with Nesterov Momentum. Much like "
                 "Adam but uses a different formula for calculating momentum."
                 "\n\t rms-prop - Root Mean Square Propogation. Maintains a moving (discounted) "
                 "average of the square of the gradients. Divides the gradient by the root of "
                 "this average.")
        self.add_item(
            section=section,
            title="learning_rate",
            datatype=float,
            default=5e-5,
            min_max=(1e-6, 1e-4),
            rounding=6,
            fixed=False,
            group="optimizer",
            info="Learning rate - how fast your network will learn (how large are the "
                 "modifications to the model weights after one batch of training). Values that "
                 "are too large might result in model crashes and the inability of the model to "
                 "find the best solution. Values that are too small might be unable to escape "
                 "from dead-ends and find the best global minimum.")
        self.add_item(
            section=section,
            title="reflect_padding",
            datatype=bool,
            default=False,
            group="network",
            info="Use reflection padding rather than zero padding with convolutions. "
                 "Each convolution must pad the image boundaries to maintain the proper "
                 "sizing. More complex padding schemes can reduce artifacts at the "
                 "border of the image."
                 "\n\t http://www-cs.engr.ccny.cuny.edu/~wolberg/cs470/hw/hw2_pad.txt")
        self.add_item(
            section=section,
            title="allow_growth",
            datatype=bool,
            default=False,
            group="network",
            fixed=False,
            info="[Nvidia Only]. Enable the Tensorflow GPU 'allow_growth' configuration option. "
                 "This option prevents Tensorflow from allocating all of the GPU VRAM at launch "
                 "but can lead to higher VRAM fragmentation and slower performance. Should only "
                 "be enabled if you are receiving errors regarding 'cuDNN fails to initialize' "
                 "when commencing training.") </s> remove             section=section, title="icnr_init", datatype=bool,
            default=False, group="initialization",
            info="Use ICNR to tile the default initializer in a repeating pattern. "
                 "This strategy is designed for pairing with sub-pixel / pixel shuffler "
                 "to reduce the 'checkerboard effect' in image reconstruction. "
                 "\n\t https://arxiv.org/ftp/arxiv/papers/1707/1707.02937.pdf")
        self.add_item(
            section=section, title="conv_aware_init", datatype=bool,
            default=False, group="initialization",
            info="Use Convolution Aware Initialization for convolutional layers. "
                 "This can help eradicate the vanishing and exploding gradient problem "
                 "as well as lead to higher accuracy, lower loss and faster convergence.\nNB:"
                 "\n\t This can use more VRAM when creating a new model so you may want to "
                 "lower the batch size for the first run. The batch size can be raised "
                 "again when reloading the model. "
                 "\n\t Multi-GPU is not supported for this option, so you should start the model "
                 "on a single GPU. Once training has started, you can stop training, enable "
                 "multi-GPU and resume."
                 "\n\t Building the model will likely take several minutes as the calculations "
                 "for this initialization technique are expensive. This will only impact starting "
                 "a new model.")
        self.add_item(
            section=section, title="reflect_padding", datatype=bool,
            default=False, group="network",
            info="Use reflection padding rather than zero padding with convolutions. "
                 "Each convolution must pad the image boundaries to maintain the proper "
                 "sizing. More complex padding schemes can reduce artifacts at the "
                 "border of the image."
                 "\n\t http://www-cs.engr.ccny.cuny.edu/~wolberg/cs470/hw/hw2_pad.txt")
        self.add_item(
            section=section, title="allow_growth", datatype=bool, default=False, group="network",
            fixed=False,
            info="[Nvidia Only]. Enable the Tensorflow GPU 'allow_growth' configuration option. "
                 "This option prevents Tensorflow from allocating all of the GPU VRAM at launch "
                 "but can lead to higher VRAM fragmentation and slower performance. Should only "
                 "be enabled if you are receiving errors regarding 'cuDNN fails to initialize' "
                 "when commencing training.")
        self.add_item(
            section=section, title="penalized_mask_loss", datatype=bool,
            default=True, group="loss",
 </s> add             section=section,
            title="penalized_mask_loss",
            datatype=bool,
            default=True,
            group="loss", </s> remove             section=section, title="mask_type", datatype=str, default="extended",
            choices=PluginLoader.get_available_extractors("mask", add_none=True), group="mask",
 </s> add             section=section,
            title="mask_type",
            datatype=str,
            default="extended",
            choices=PluginLoader.get_available_extractors("mask", add_none=True),
            group="mask", </s> remove             section=section, title="mask_blur_kernel", datatype=int, min_max=(0, 9),
            rounding=1, default=3, group="mask",
 </s> add             section=section,
            title="mask_blur_kernel",
            datatype=int,
            min_max=(0, 9),
            rounding=1,
            default=3,
            group="mask",
          if not self._is_predict:
              self._state.add_session_loss_names(loss.names)
          logger.debug("Compiled Model: %s", self._model)
  











































      def _legacy_mapping(self):  # pylint:disable=no-self-use
          """ The mapping of separate model files to single model layers for transferring of legacy
          weights.
  
          Returns
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 training config - Add selectable optimizers 
plugins/train/model/_base.py