code: def _update(iter, config, cost, scheduler):
      comm = current_communicator()
  
      loaded_data = {}
      is_first_optimizer = True
  
      def _sum_cost():
          if comm:
              # logger.log(99, "Calc cost with communicator")
              var = [nn.NdArray()]
              var[0].data = cost.sum_iteration
              _all_reduce(comm, var, division=False, inplace=True)
              cost.sum_epoch += var[0].data
              cost.num_iteration += comm.size
          else:
              cost.sum_epoch += cost.sum_iteration
              cost.num_iteration += 1
  
      def _get_reserved_variable(
          shape, reserved_variable_name, iter, iter_per_epoch, max_epoch
      ):
          if reserved_variable_name == "%iter":
              value = iter
          elif reserved_variable_name == "%max_iter":
              value = max_epoch * iter_per_epoch
          elif reserved_variable_name == "%epoch":
              value = iter // iter_per_epoch
          elif reserved_variable_name == "%epochf":
              value = iter * 1.0 / iter_per_epoch
          elif reserved_variable_name == "%max_epoch":
              value = max_epoch
          elif reserved_variable_name == "%progress":
              value = (iter * 1.0 / iter_per_epoch) / max_epoch
          else:
              raise ValueError(
                  "Unknown reserved variable {}".format(
                      reserved_variable_name
                  )
              )
          return value
  
      for opt in config.optimizers.values():
          o = opt.optimizer
          if (o.start_iter == 0 or iter + 1 >= o.start_iter) and (
              o.end_iter == 0 or iter + 1 <= o.end_iter
          ):
              # Load dataset
              data = OrderedDict()
              for di in opt.data_iterators:
                  if di not in loaded_data:
                      loaded_data[di] = di.next()
                  data.update(zip(di.variables, loaded_data[di]))
              for v, d in o.dataset_assign.items():
                  # TODO: here we consume a bit more memory for loading all edge nodes to cuda
                  # dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                  #     0].inputs else None
                  dest_context = config.global_config.default_context
                  if d not in data and d[0] == "%":
                      value = _get_reserved_variable(
                          v.variable_instance.shape,
                          d,
                          iter,
                          config.training_config.iter_per_epoch,
                          config.training_config.max_epoch,
                      )
                      v.variable_instance.data.fill(value)
                  elif d in data:
                      let_data_to_variable(
                          v.variable_instance,
                          data[d],
                          ctx=dest_context,
                          data_name=d,
                          variable_name=v.name,
                      )
                  else:
                      raise ValueError(
                          'Variable "{}" is not found in dataset "{}", optimizer "{}"'.format(
                              d,
                              ", ".join(o.data_iterators.keys()),
                              o.name,
                          )
                      )
  
              # Generate data
              for v, generator in o.generator_assign.items():
                  # TODO: here we consume a bit more memory for loading all edge nodes to cuda
                  # dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                  #     0].inputs else None
                  dest_context = config.global_config.default_context
                  let_data_to_variable(
                      v.variable_instance,
                      data=generator(v.variable_instance.d.shape),
                      ctx=dest_context,
                      variable_name=v.name,
                  )
  
              # Monitor loss before forward to prepare input data while processing on
              # GPU
              if cost.variables:
                  for l in cost.variables:
                      cost.sum_iteration += np.mean(
                          l.variable_instance.d
                      )
                      # l.variable_instance.data.zero()
                  if is_first_optimizer:
                      is_first_optimizer = False
                      _sum_cost()
                      if single_or_rankzero():
                          progress(
                              "Training : cost={0:0.6f}".format(
                                  cost.sum_iteration
                              ),
                              (
                                  iter
                                  % config.training_config.iter_per_epoch
                              )
                              * 1.0
                              / config.training_config.iter_per_epoch,
                          )
                      cost.sum_iteration = 0.0
  
              with scheduler:
                  # This size is omitted if multiple variables in loss_variables
                  # otherwise, this size is omitted since passing through F.sink()
                  l_size = o.loss_variables[0].variable_instance.size
  
                  with nodeTimeCollector.collect_cost_time(comm, iter):
                      # Forward
                      o.target.forward(clear_no_need_grad=True)
  
                      # Equivalency with previous version
                      if iter % o.update_interval == 0:
                          o.solver.zero_grad()
  
                      # Backward
                      if (
                          o.comm
                          and iter % o.update_interval
                          == o.update_interval - 1
                      ):
                          params = [
                              x.grad for x in o.parameters.values()
                          ]
                          o.target.backward(
                              grad=1.0 / l_size,
                              clear_buffer=True,
                              communicator_callbacks=comm.all_reduce_callback(
                                  params, 2 << 20, division=True
                              ),
                          )
                      else:
                          o.target.backward(
                              grad=1.0 / l_size, clear_buffer=True
                          )
  
                  # Update
                  if iter % o.update_interval == o.update_interval - 1:
                      if o.weight_decay > 0:
                          o.solver.weight_decay(o.weight_decay)
  
                      if o.scheduler is not None:
                          o.solver.set_learning_rate(
                              o.scheduler.get_learning_rate(iter)
                          )
                      o.solver.update()
  
              # Reserve monitor loss
              cost.variables = o.loss_variables
  
      # Monitor loss at the end of epoch
      if (
          iter % config.training_config.iter_per_epoch
          == config.training_config.iter_per_epoch - 1
          and cost.variables
      ):
          for l in cost.variables:
              cost.sum_iteration += np.mean(l.variable_instance.d)
              # l.variable_instance.data.zero()
          _sum_cost()
          cost.variables = None
          cost.sum_iteration = 0.0
  
      return cost
  
imports: import glob
  import os
  import time
  import zipfile
  from collections import OrderedDict
  import nnabla
  import nnabla.utils.callback
  import nnabla.utils.load
  import numpy
  from contextlib2 import ExitStack
  from nnabla import available_contexts
  from nnabla.config import nnabla_config
  from nnabla.logger import logger
  from nnabla.parameter import save_parameters
  from nnabla.utils.cli.utility import NodeTimeInfoCollector
  from nnabla.utils.cli.utility import get_cpu_gpu_average_load
  from nnabla.utils.cli.utility import let_data_to_variable
  from nnabla.utils.cli.utility import lms_scheduler
  from nnabla.utils.cli.utility import load_train_state
  from nnabla.utils.cli.utility import measure_cpu_gpu_instant_load
  from nnabla.utils.cli.utility import save_optimizer_states
  from nnabla.utils.cli.utility import str_to_num
  from nnabla.utils.communicator_util import current_communicator, single_or_rankzero
  from nnabla.utils.nnp_format import nnp_version
  from nnabla.utils.progress import configure_progress, progress
  from six.moves import range
label_desc: Use image node_id 3 for calling a global function with example usage: _update(iter, config, cost, scheduler) and returns: cost, value