code: def compare_optimizer(
      config, parameters, config_cpu, parameters_cpu, result_array
  ):
      loaded_data = {}
      for opt, opt_cpu in zip(
          config.optimizers.values(), config_cpu.optimizers.values()
      ):
          o = opt.optimizer
          o_cpu = opt_cpu.optimizer
          opts = [o, o_cpu]
  
          result_name = "optimizer '%s' with network '%s'" % (
              o.name,
              o.network.name,
          )
          result_dict = OrderedDict()
  
          logger.log(99, "Comparing " + result_name + " ...")
          logger.log(
              99,
              "process(func, variable), norm_diff, current_context_std, cpu_std, diff_std",
          )
          # Start comparison with same parameters
          for p, p_cpu in zip(
              parameters.values(), parameters_cpu.values()
          ):
              p_cpu.d = p.d
  
          # Load dataset
          di = opt.data_iterator
          if di not in loaded_data:
              loaded_data[di] = di.next()
          data = loaded_data[di]
  
          for v, d in o.dataset_assign.items():
              let_data_to_variable(
                  v.variable_instance,
                  data[di.variables.index(d)],
                  data_name=d,
                  variable_name=v.name,
              )
          for v, d in o_cpu.dataset_assign.items():
              let_data_to_variable(
                  v.variable_instance,
                  data[di.variables.index(d)],
                  data_name=d,
                  variable_name=v.name,
              )
  
          # Generate data
          generated = {}
          for v, generator in o.generator_assign.items():
              generated[v.name] = generator(v.shape)
              dest_context = (
                  config.global_config.default_context
                  if not o.forward_sequence
                  or v not in o.forward_sequence[0].inputs
                  else None
              )
              let_data_to_variable(
                  v.variable_instance,
                  data=generated[v.name],
                  ctx=dest_context,
                  variable_name=v.name,
              )
          for v, generator in o_cpu.generator_assign.items():
              dest_context = (
                  config.global_config.default_context
                  if not o.forward_sequence
                  or v not in o.forward_sequence[0].inputs
                  else None
              )
              let_data_to_variable(
                  v.variable_instance,
                  data=generated[v.name],
                  ctx=dest_context,
                  variable_name=v.name,
              )
  
          last_max_diff = 1e-5
  
          # Forward
          for func, func_cpu in zip(
              o.forward_sequence, o_cpu.forward_sequence
          ):
              o.network.forward_function(func)
              o_cpu.network.forward_function(func_cpu)
              large_diff = False
              for v, v_cpu in zip(func.outputs, func_cpu.outputs):
                  name = "forward_function (%s, %s)" % (
                      func.name,
                      v.name,
                  )
                  if (
                      v.variable_instance.d.shape
                      != v_cpu.variable_instance.d.shape
                  ):
                      logger.log(
                          99,
                          "Variable shape is different in %s (current_context=%s, cpu=%s)"
                          % (
                              v.name,
                              str(v.variable_instance.d.shape),
                              str(v_cpu.variable_instance.d.shape),
                          ),
                      )
                  norm_diff, std1, std2, diff_std = calc_norm_diff(
                      v.variable_instance.d, v_cpu.variable_instance.d
                  )
                  logger.log(
                      99,
                      "%s, %f, %f, %f, %f"
                      % (name, norm_diff, std1, std2, diff_std),
                  )
                  result_dict[name] = norm_diff
                  if norm_diff > last_max_diff:
                      if norm_diff > last_max_diff * 10:
                          logger.log(
                              99,
                              "  current_context(data)="
                              + str(v.variable_instance.d.flatten()),
                          )
                          logger.log(
                              99,
                              "  cpu(data)="
                              + str(
                                  v_cpu.variable_instance.d.flatten()
                              ),
                          )
                          large_diff = True
                      last_max_diff = norm_diff
              if large_diff:
                  logger.log(99, "  x_data:")
                  for v, v_cpu in zip(func.inputs, func_cpu.inputs):
                      logger.log(
                          99,
                          "    current_context(%s.d)=%s"
                          % (
                              v.name,
                              str(v.variable_instance.d.flatten()),
                          ),
                      )
                      logger.log(
                          99,
                          "    cpu(%s.d)=%s"
                          % (
                              v_cpu.name,
                              str(v_cpu.variable_instance.d.flatten()),
                          ),
                      )
  
          # Backward
          o.network.prepare_backward(o.backward_sequence)
          o_cpu.network.prepare_backward(o_cpu.backward_sequence)
          for seq, seq_cpu in zip(
              o.backward_sequence.sequence,
              o_cpu.backward_sequence.sequence,
          ):
              o.network.backward_function(seq)
              o_cpu.network.backward_function(seq_cpu)
              large_diff = False
              for v, v_cpu in zip(seq.func.inputs, seq_cpu.func.inputs):
                  if v.variable_instance.need_grad:
                      name = "backward_function (%s, %s)" % (
                          seq.func.name,
                          v.name,
                      )
                      norm_diff, std1, std2, diff_std = calc_norm_diff(
                          v.variable_instance.g,
                          v_cpu.variable_instance.g,
                      )
                      logger.log(
                          99,
                          "%s, %f, %f, %f, %f"
                          % (name, norm_diff, std1, std2, diff_std),
                      )
                      result_dict[name] = norm_diff
                      if norm_diff > last_max_diff:
                          if norm_diff > last_max_diff * 10:
                              logger.log(
                                  99,
                                  "  current_context(diff)="
                                  + str(v.variable_instance)
                                  + str(
                                      v.variable_instance.g.flatten()
                                  ),
                              )
                              logger.log(
                                  99,
                                  "  cpu(diff)="
                                  + str(v_cpu.variable_instance)
                                  + str(
                                      v_cpu.variable_instance.g.flatten()
                                  ),
                              )
                              large_diff = True
                          last_max_diff = norm_diff
              if large_diff:
                  logger.log(99, "  x_data:")
                  for v, v_cpu in zip(
                      seq.func.inputs, seq_cpu.func.inputs
                  ):
                      logger.log(
                          99,
                          "    current_context(%s.d)=%s"
                          % (
                              v.name,
                              str(v.variable_instance.d.flatten()),
                          ),
                      )
                      logger.log(
                          99,
                          "    cpu(%s.d)=%s"
                          % (
                              v_cpu.name,
                              str(v_cpu.variable_instance.d.flatten()),
                          ),
                      )
                  logger.log(99, "  y_diff:")
                  for v, v_cpu in zip(
                      seq.func.outputs, seq_cpu.func.outputs
                  ):
                      logger.log(
                          99,
                          "    current_context(%s.g)=%s"
                          % (
                              v.name,
                              str(v.variable_instance.g.flatten()),
                          ),
                      )
                      logger.log(
                          99,
                          "    cpu(%s.g)=%s"
                          % (
                              v_cpu.name,
                              str(v_cpu.variable_instance.g.flatten()),
                          ),
                      )
  
          # Update (weight decay)
          if o.weight_decay > 0:
              o.solver.weight_decay(o.weight_decay)
              o_cpu.solver.weight_decay(o_cpu.weight_decay)
  
          # Update
          o.solver.update()
          o_cpu.solver.update()
          for i, ((v, lr), (v_cpu, lr_cpu)) in enumerate(
              zip(
                  o.parameter_learning_rate_multipliers.items(),
                  o_cpu.parameter_learning_rate_multipliers.items(),
              )
          ):
              if lr > 0:
                  name = "update (%s, %s)" % (o.solver.name, v.name)
                  norm_diff, std1, std2, diff_std = calc_norm_diff(
                      v.variable_instance.d, v_cpu.variable_instance.d
                  )
                  logger.log(
                      99,
                      "%s, %f, %f, %f, %f"
                      % (name, norm_diff, std1, std2, diff_std),
                  )
                  result_dict[name] = norm_diff
  
          result_array = add_result(
              result_name, result_dict, result_array
          )
  
      return result_array
  
file_path: nnabla/python/src/nnabla/utils/cli/compare_with_cpu.py
imports: import os
  from collections import OrderedDict
  import nnabla
  import nnabla.utils.load
  import numpy
  from contextlib2 import ExitStack
  from nnabla.logger import logger
  from nnabla.parameter import get_parameters
  from nnabla.utils.cli.utility import let_data_to_variable
  from nnabla.utils.progress import configure_progress, progress
label_desc: Use image node_id 3 for calling a global function with example usage: compare_optimizer(config, parameters, config_cpu, parameters_cpu, result_array) and returns: result_array