class_docstr: Parallel processing engine.
  
  Engine for running computational graph.
  
  :class:`ParallelProcessing` is a manager for executing a sequence of tasks
  in a given caller, where each task is run sequentially, but assumed to be
  parallelized internally. The main responsibility of
  :class:`ParallelProcessing` is to handle memory-mapping, estimation
  cache updates, input and output array updates and output collection.
  
  Parameters
  ----------
  caller :  obj
      the caller of the job. Either a Layer or a meta layer class
      such as Sequential.
  
  *args : optional
      Optional arguments to :class:`~mlens.parallel.backend.BaseProcessor`
  
  **kwargs : optional
      Optional keyword arguments to
      :class:`~mlens.parallel.backend.BaseProcessor`.
class_docstr_tok: ["Parallel","processing","engine",".","Engine","for","running","computational","graph",".",":","class",":","`","ParallelProcessing","`","is","a","manager","for","executing","a","sequence","of","tasks","in","a","given","caller",",","where","each","task","is","run","sequentially",",","but","assumed","to","be","parallelized","internally",".","The","main","responsibility","of",":","class",":","`","ParallelProcessing","`","is","to","handle","memory-mapping",",","estimation","cache","updates",",","input","and","output","array","updates","and","output","collection",".","Parameters","--","--","--","--","--","caller",":","obj","the","caller","of","the","job",".","Either","a","Layer","or","a","meta","layer","class","such","as","Sequential",".","*","args",":","optional","Optional","arguments","to",":","class",":","`","~mlens.parallel.backend.BaseProcessor","`","*","*","kwargs",":","optional","Optional","keyword","arguments","to",":","class",":","`","~mlens.parallel.backend.BaseProcessor","`","."]
docstr: Process job.
  
  Main method for processing a caller. Requires the instance to be
  setup by a prior call to
  :func:`~mlens.parallel.backend.BaseProcessor.initialize`.
  
  .. seealso::
      :func:`~mlens.parallel.backend.ParallelProcessing.map`,
      :func:`~mlens.parallel.backend.ParallelProcessing.stack`
  
  Parameters
  ----------
  caller : iterable
      Iterable that generates accepted task instances. Caller should be
      a child of the :class:`~mlens.parallel.base.BaseBackend` class,
      and tasks need to implement an appropriate call method.
  
  out : dict
      A dictionary with output parameters. Pass an empty dict for no
      output. See
      :func:`~mlens.parallel.backend.BaseProcessor.initialize` for more
      details.
  
  Returns
  -------
  out: array-like, list, optional
      Prediction array(s).
docstr_tok: ["Process","job",".","Main","method","for","processing","a","caller",".","Requires","the","instance","to","be","setup","by","a","prior","call","to",":","func",":","`","~mlens.parallel.backend.BaseProcessor.initialize","`",".","..","seealso",":",":",":","func",":","`","~mlens.parallel.backend.ParallelProcessing.map","`",",",":","func",":","`","~mlens.parallel.backend.ParallelProcessing.stack","`","Parameters","--","--","--","--","--","caller",":","iterable","Iterable","that","generates","accepted","task","instances",".","Caller","should","be","a","child","of","the",":","class",":","`","~mlens.parallel.base.BaseBackend","`","class",",","and","tasks","need","to","implement","an","appropriate","call","method",".","out",":","dict","A","dictionary","with","output","parameters",".","Pass","an","empty","dict","for","no","output",".","See",":","func",":","`","~mlens.parallel.backend.BaseProcessor.initialize","`","for","more","details",".","Returns","--","--","--","-","out",":","array-like",",","list",",","optional","Prediction","array","(","s",")","."]
code: def process(self, caller, out, **kwargs):
      """Process job.
  
      Main method for processing a caller. Requires the instance to be
      setup by a prior call to
      :func:`~mlens.parallel.backend.BaseProcessor.initialize`.
  
      .. seealso::
          :func:`~mlens.parallel.backend.ParallelProcessing.map`,
          :func:`~mlens.parallel.backend.ParallelProcessing.stack`
  
      Parameters
      ----------
      caller : iterable
          Iterable that generates accepted task instances. Caller should be
          a child of the :class:`~mlens.parallel.base.BaseBackend` class,
          and tasks need to implement an appropriate call method.
  
      out : dict
          A dictionary with output parameters. Pass an empty dict for no
          output. See
          :func:`~mlens.parallel.backend.BaseProcessor.initialize` for more
          details.
  
      Returns
      -------
      out: array-like, list, optional
          Prediction array(s).
      """
      check_initialized(self)
  
      return_names = out.pop("return_names", [])
      return_final = out.pop("return_final", False)
      out = list() if return_names else None
  
      tf = self.job.dir if not isinstance(self.job.dir, list) else None
      with Parallel(
          n_jobs=self.n_jobs,
          temp_folder=tf,
          max_nbytes=None,
          mmap_mode="w+",
          verbose=self.verbose,
          backend=self.backend,
      ) as parallel:
          for task in caller:
              self.job.clear()
  
              self._partial_process(task, parallel, **kwargs)
  
              if task.name in return_names:
                  out.append(self.get_preds(dtype=_dtype(task)))
  
              self.job.update()
  
      if return_final:
          out = self.get_preds(dtype=_dtype(task))
      return out
  
imports: from __future__ import with_statement, division
  import gc
  import os
  import shutil
  import subprocess
  import tempfile
  import warnings
  from abc import ABCMeta, abstractmethod
  import numpy
  from scipy.sparse import issparse, hstack
  from ..None import config
  from ..externals.joblib import Parallel, dump, load
  from ..utils import check_initialized
  from ..utils.exceptions import ParallelProcessingError, ParallelProcessingWarning
  from ..externals.sklearn.validation import check_random_state
label_desc: Use image node_id 4 for calling the ParallelProcessing obj's underlying member method code with example usage: obj.process(caller, out) and returns: out