code: def profile_optimizer(config, result_array, synchronize):
      # Profile Training
      for opt in config.optimizers.values():
          o = opt.optimizer
          result_name = "optimizer '%s' with network '%s'" % (
              o.name,
              o.network.name,
          )
          result_dict = OrderedDict()
  
          logger.log(99, "Profiling " + result_name + " ...")
          # Clear weight
          for name, p in o.parameters.items():
              if name[-2:] in ("/W", "/b"):
                  p.data.zero()
  
          # Load dataset
          def load_dataset():
              loaded_data = {}
              data = OrderedDict()
              for di in opt.data_iterators:
                  if di not in loaded_data:
                      loaded_data[di] = di.next()
                  data.update(zip(di.variables, loaded_data[di]))
              return data
  
          profile(
              config,
              "load_dataset",
              load_dataset,
              result_dict,
              synchronize,
          )
  
          # Let data
          data = load_dataset()
          for v, d in o.dataset_assign.items():
  
              def let_data():
                  if d not in data:
                      raise ValueError(
                          'Data "' + d + '" is not found in dataset.'
                      )
                  let_data_to_variable(
                      v.variable_instance,
                      data=data[d],
                      data_name=d,
                      variable_name=v.name,
                  )
  
              profile(
                  config,
                  "let_data (%s to %s)" % (d, v.name),
                  let_data,
                  result_dict,
                  synchronize,
              )
  
          # Generate data
          for v, generator in o.generator_assign.items():
  
              def generate_data():
                  let_data_to_variable(
                      v.variable_instance,
                      data=generator(v.variable_instance.d.shape),
                      variable_name=v.name,
                  )
  
              profile(
                  config,
                  "generate_data (%s)" % v.name,
                  generate_data,
                  result_dict,
                  synchronize,
              )
  
          """
          # Setup (detail)
          for func in o.forward_sequence:
              def setup():
                  o.network.setup_function(func)
              profile(config, 'setup_function (%s : %s)' % (
                  func.name, func.function_instance.name), setup, result_dict, synchronize)
          """
          # Warm-up
          # o.network.forward(o.forward_sequence)
          # o.network.prepare_backward(o.backward_sequence)
          # o.network.backward(o.backward_sequence)
          o.forward_sequence = []
          o.backward_sequence = []
          o.target.forward(
              clear_no_need_grad=True,
              function_pre_hook=lambda f: o.forward_sequence.append(f),
          )
          o.target.backward(
              clear_buffer=True,
              function_pre_hook=lambda f: o.backward_sequence.append(f),
          )
  
          # Forward (detail)
          for func in o.forward_sequence:
              if func.name == "Sink":
                  continue
              profile(
                  config,
                  "forward_function (%s : %s)" % (func.name, func.name),
                  partial(
                      func.forward,
                      inputs=func.inputs,
                      outputs=func.outputs,
                  ),
                  result_dict,
                  synchronize,
              )
  
          # Backward (detail)
          def empty_func():
              pass  # keep this for backward compatible
  
          profile(
              config,
              "prepare_backward",
              empty_func,
              result_dict,
              synchronize,
          )
  
          for func in o.backward_sequence:
              if func.name == "Sink":
                  continue
              profile(
                  config,
                  "backward_function (%s : %s)"
                  % (func.name, func.name),
                  partial(
                      func.backward,
                      inputs=func.inputs,
                      outputs=func.outputs,
                  ),
                  result_dict,
                  synchronize,
              )
  
          # Forward (all)
          def forward_all():
              o.target.forward(clear_no_need_grad=True)
  
          profile(
              config,
              "forward_all",
              forward_all,
              result_dict,
              synchronize,
          )
  
          # Backward (all)
          def backward_all():
              o.target.backward(clear_buffer=True)
  
          profile(
              config,
              "backward_all",
              backward_all,
              result_dict,
              synchronize,
          )
  
          # Backward (wo param zero_grad)
          # Backward (all)
          def backward_all_wo_zero_grad():
              for name, p in o.parameters.items():
                  if name[-2:] in ("/W", "/b"):
                      p.grad.zero()
              o.target.backward(clear_buffer=True)
  
          profile(
              config,
              "backward_all(wo param zero_grad)",
              backward_all_wo_zero_grad,
              result_dict,
              synchronize,
          )
  
          # Update (weight decay)
          if o.weight_decay > 0:
  
              def weight_decay():
                  o.solver.weight_decay(o.weight_decay)
  
              profile(
                  config,
                  "weight_decay (%s)" % o.solver.name,
                  weight_decay,
                  result_dict,
                  synchronize,
              )
  
          # Update
          def update():
              o.solver.update()
  
          profile(
              config,
              "update (%s)" % o.solver.name,
              update,
              result_dict,
              synchronize,
          )
  
          # Monitor loss
          def monitor_loss():
              for l in o.loss_variables:
                  np.mean(l.variable_instance.d)
  
          profile(
              config,
              "monitor_loss",
              monitor_loss,
              result_dict,
              synchronize,
          )
  
          result_array = add_result(
              result_name, result_dict, result_array
          )
  
      return result_array
  
file_path: nnabla/python/src/nnabla/utils/cli/profile.py
imports: from __future__ import print_function
  import os
  import time
  from collections import OrderedDict
  from functools import partial
  import nnabla.utils.callback
  import nnabla.utils.load
  import numpy
  from contextlib2 import ExitStack
  from nnabla.ext_utils import import_extension_module
  from nnabla.logger import logger
  from nnabla.utils.cli.utility import let_data_to_variable
  from nnabla.utils.progress import configure_progress, progress
label_desc: Use image node_id 3 for calling a global function with example usage: profile_optimizer(config, result_array, synchronize) and returns: result_array, data