code: def create_multi_node_evaluator(actual_evaluator, communicator):
      """Create a multi node evaluator from a normal evaluator.
  
      Actually this method patches the evaluator to work in multi node
      environment. This method adds several hidden attributes starting
      with `_mn_` prefix.
  
      Args:
          actual_evaluator: evaluator to be patched
              (e.g., ``chainer.training.extensions.Evaluator``)
          communicator: ChainerMN communicator
  
      Returns:
          The multi-node patched ``actual_evaluator``.
  
      .. note:: After patched, original evaluator does not work
                correctly in non-MPI environment.
  
      """
  
      actual_evaluator._mn_original_evaluate = actual_evaluator.evaluate
      actual_evaluator._mn_communicator = communicator
  
      def new_evaluate(self):
          local_mean_dict = self._mn_original_evaluate()
  
          # ChainerX support:
          # We need convert chainerx ndarray to Native array because
          #   (1) allreduce_obj is used to compute global mean values, since
          #       a simple allreduce operation cannot be applied in evaluation.
          #   (2) allreduce_obj calls mpi4py.allreduce, which pickles the object
          #   (3) chainerx.ndarray preserves CUDA device internally when pickled
          #   (4) An error will occur when an ndarray is unpickled in another
          #       process
          arrays = list(local_mean_dict.values())
          if len(arrays) > 0:
              array0 = list(local_mean_dict.values())[0]
              xp = backend.get_array_module(array0)
              if xp == chx and array0.device.backend.name == "cuda":
                  # Results of evaluation is fairly small, so
                  # the ndarray is transferred to CPU and allreduce()-ed.
                  # NOTE: Matrices for evaluation are transferred to the
                  # host memory and sent via MPI instead of NCCL.
                  # Although evaluation matrices are small in most cases,
                  # this is a potential performance issue.
                  local_mean_dict = {
                      name: chx.to_numpy(value)
                      for name, value in local_mean_dict.items()
                  }
  
          global_mean_dict = {
              name: self._mn_communicator.allreduce_obj(value)
              / self._mn_communicator.size
              for name, value in sorted(local_mean_dict.items())
          }
          return global_mean_dict
  
      actual_evaluator.evaluate = six.create_bound_method(
          new_evaluate, actual_evaluator
      )
      return actual_evaluator
  
file_path: chainer/chainermn/extensions/multi_node_evaluator.py
imports: import copy
  import six
  from chainer.training import extension
  from chainer import backend
  from chainer.dataset import convert
  from chainer import function
  from chainer.utils import argument
  import chainerx
label_desc: Use image node_id 1 for calling a global function with example usage: create_multi_node_evaluator(actual_evaluator, communicator) and returns: actual_evaluator, global_mean_dict