sidpy/proc/comp_utils.py
def parallel_compute(data, func, cores=None, lengthy_computation=False,
                     func_args=None, func_kwargs=None, verbose=False,
                     joblib_backend='multiprocessing'):
    """
    Computes the provided function using multiple cores using the joblib
    library

    Parameters
    ----------
    data : numpy.ndarray
        Data to map function to. Function will be mapped to the first axis of
        data
    func : callable
        Function to map to data
    cores : uint, optional
        Number of logical cores to use to compute
        Default - All cores - 1 (total cores <= 4) or - 2 (cores > 4) depending
        on number of cores.
        Ignored in the MPI context - each rank will execute serially
    lengthy_computation : bool, optional
        Whether or not each computation is expected to take substantial time.
        Sometimes the time for adding more cores can outweigh the time per core
        Default - False
    func_args : list, optional
        arguments to be passed to the function
    func_kwargs : dict, optional
        keyword arguments to be passed onto function
    joblib_backend : str, optional
        Backend to use for parallel computation with Joblib.
        The older paradigm - "multiprocessing" is the default in pyUSID.
        Set to None to use the joblib default - "loky"
    verbose : bool, optional. default = False
        Whether or not to print statements that aid in debugging

    Returns
    -------
    results : list
        List of computational results
    """

    if not callable(func):
        raise TypeError('Function argument is not callable')
    if not isinstance(data, np.ndarray):
        raise TypeError('data must be a numpy array')
    if func_args is None:
        func_args = list()
    else:
        if isinstance(func_args, tuple):
            func_args = list(func_args)
        if not isinstance(func_args, list):
            raise TypeError('Arguments to the mapped function should be '
                            'specified as a list')
    if func_kwargs is None:
        func_kwargs = dict()
    else:
        if not isinstance(func_kwargs, dict):
            raise TypeError('Keyword arguments to the mapped function should '
                            'be specified via a dictionary')

    req_cores = cores
    MPI = get_MPI()
    if MPI is not None:
        rank = MPI.COMM_WORLD.Get_rank()
        # Was unable to get the MPI + joblib framework to work.
        # Did not compute anything at all. Just froze
        cores = 1
    else:
        rank = 0
        cores = recommend_cpu_cores(data.shape[0],
                                    requested_cores=cores,
                                    lengthy_computation=lengthy_computation,
                                    verbose=verbose)

    if verbose:
        print('Rank {} starting computing on {} cores (requested {} cores)'
              ''.format(rank, cores, req_cores))

    if cores > 1:
        values = [joblib.delayed(func)(x, *func_args, **func_kwargs) for x in data]
        results = joblib.Parallel(n_jobs=cores, backend=joblib_backend)(values)

        # Finished reading the entire data set
        print('Rank {} finished parallel computation'.format(rank))

    else:
        if verbose:
            print("Rank {} computing serially ...".format(rank))
        # List comprehension vs map vs for loop?
        # https://stackoverflow.com/questions/1247486/python-list-comprehension-vs-map
        results = [func(vector, *func_args, **func_kwargs) for vector in data]

    return results

    Computes the provided function using multiple cores using the joblib
    library

    Parameters
    ----------
    data : numpy.ndarray
        Data to map function to. Function will be mapped to the first axis of
        data
    func : callable
        Function to map to data
    cores : uint, optional
        Number of logical cores to use to compute
        Default - All cores - 1 (total cores <= 4) or - 2 (cores > 4) depending
        on number of cores.
        Ignored in the MPI context - each rank will execute serially
    lengthy_computation : bool, optional
        Whether or not each computation is expected to take substantial time.
        Sometimes the time for adding more cores can outweigh the time per core
        Default - False
    func_args : list, optional
        arguments to be passed to the function
    func_kwargs : dict, optional
        keyword arguments to be passed onto function
    joblib_backend : str, optional
        Backend to use for parallel computation with Joblib.
        The older paradigm - "multiprocessing" is the default in pyUSID.
        Set to None to use the joblib default - "loky"
    verbose : bool, optional. default = False
        Whether or not to print statements that aid in debugging

    Returns
    -------
    results : list
        List of computational results
    
Computes the provided function using multiple cores using the joblib
library

Parameters

data : numpy.ndarray
Data to map function to. Function will be mapped to the first axis of
data
func : callable
Function to map to data
cores : uint, optional
Number of logical cores to use to compute
Default - All cores - 1 (total cores <= 4) or - 2 (cores > 4) depending
on number of cores.
Ignored in the MPI context - each rank will execute serially
lengthy_computation : bool, optional
Whether or not each computation is expected to take substantial time.
Sometimes the time for adding more cores can outweigh the time per core
Default - False
func_args : list, optional
arguments to be passed to the function
func_kwargs : dict, optional
keyword arguments to be passed onto function
joblib_backend : str, optional
Backend to use for parallel computation with Joblib.
The older paradigm - "multiprocessing" is the default in pyUSID.
Set to None to use the joblib default - "loky"
verbose : bool, optional. default = False
Whether or not to print statements that aid in debugging

Returns

results : list
List of computational results
Computes the provided function using multiple cores using the joblib library Parameters data : numpy . ndarray Data to map function to . Function will be mapped to the first axis of data func : callable Function to map to data cores : uint optional Number of logical cores to use to compute Default - All cores - 1 ( total cores < = 4 ) or - 2 ( cores > 4 ) depending on number of cores . Ignored in the MPI context - each rank will execute serially lengthy_computation : bool optional Whether or not each computation is expected to take substantial time . Sometimes the time for adding more cores can outweigh the time per core Default - False func_args : list optional arguments to be passed to the function func_kwargs : dict optional keyword arguments to be passed onto function joblib_backend : str optional Backend to use for parallel computation with Joblib . The older paradigm - " multiprocessing " is the default in pyUSID . Set to None to use the joblib default - " loky " verbose : bool optional . default = False Whether or not to print statements that aid in debugging Returns results : list List of computational results
def parallel_compute(data, func, cores=None, lengthy_computation=False,
                     func_args=None, func_kwargs=None, verbose=False,
                     joblib_backend='multiprocessing'):
    if not callable(func):
        raise TypeError('Function argument is not callable')
    if not isinstance(data, np.ndarray):
        raise TypeError('data must be a numpy array')
    if func_args is None:
        func_args = list()
    else:
        if isinstance(func_args, tuple):
            func_args = list(func_args)
        if not isinstance(func_args, list):
            raise TypeError('Arguments to the mapped function should be '
                            'specified as a list')
    if func_kwargs is None:
        func_kwargs = dict()
    else:
        if not isinstance(func_kwargs, dict):
            raise TypeError('Keyword arguments to the mapped function should '
                            'be specified via a dictionary')
    req_cores = cores
    MPI = get_MPI()
    if MPI is not None:
        rank = MPI.COMM_WORLD.Get_rank()
        cores = 1
    else:
        rank = 0
        cores = recommend_cpu_cores(data.shape[0],
                                    requested_cores=cores,
                                    lengthy_computation=lengthy_computation,
                                    verbose=verbose)
    if verbose:
        print('Rank {} starting computing on {} cores (requested {} cores)'
              ''.format(rank, cores, req_cores))
    if cores > 1:
        values = [joblib.delayed(func)(x, *func_args, **func_kwargs) for x in data]
        results = joblib.Parallel(n_jobs=cores, backend=joblib_backend)(values)
        print('Rank {} finished parallel computation'.format(rank))
    else:
        if verbose:
            print("Rank {} computing serially ...".format(rank))
        results = [func(vector, *func_args, **func_kwargs) for vector in data]
    return results
def parallel_compute ( data , func , cores = None , lengthy_computation = False , func_args = None , func_kwargs = None , verbose = False , joblib_backend = 'multiprocessing' ) : if not callable ( func ) : raise TypeError ( 'Function argument is not callable' ) if not isinstance ( data , np . ndarray ) : raise TypeError ( 'data must be a numpy array' ) if func_args is None : func_args = list ( ) else : if isinstance ( func_args , tuple ) : func_args = list ( func_args ) if not isinstance ( func_args , list ) : raise TypeError ( 'Arguments to the mapped function should be ' 'specified as a list' ) if func_kwargs is None : func_kwargs = dict ( ) else : if not isinstance ( func_kwargs , dict ) : raise TypeError ( 'Keyword arguments to the mapped function should ' 'be specified via a dictionary' ) req_cores = cores MPI = get_MPI ( ) if MPI is not None : rank = MPI . COMM_WORLD . Get_rank ( ) cores = 1 else : rank = 0 cores = recommend_cpu_cores ( data . shape [ 0 ] , requested_cores = cores , lengthy_computation = lengthy_computation , verbose = verbose ) if verbose : print ( 'Rank {} starting computing on {} cores (requested {} cores)' '' . format ( rank , cores , req_cores ) ) if cores > 1 : values = [ joblib . delayed ( func ) ( x , * func_args , ** func_kwargs ) for x in data ] results = joblib . Parallel ( n_jobs = cores , backend = joblib_backend ) ( values ) print ( 'Rank {} finished parallel computation' . format ( rank ) ) else : if verbose : print ( "Rank {} computing serially ..." . format ( rank ) ) results = [ func ( vector , * func_args , ** func_kwargs ) for vector in data ] return results
Computes the provided function using multiple cores using the joblib
library
Computes the provided function using multiple cores using the joblib library
"""
    Computes the provided function using multiple cores using the joblib
    library

    Parameters
    ----------
    data : numpy.ndarray
        Data to map function to. Function will be mapped to the first axis of
        data
    func : callable
        Function to map to data
    cores : uint, optional
        Number of logical cores to use to compute
        Default - All cores - 1 (total cores <= 4) or - 2 (cores > 4) depending
        on number of cores.
        Ignored in the MPI context - each rank will execute serially
    lengthy_computation : bool, optional
        Whether or not each computation is expected to take substantial time.
        Sometimes the time for adding more cores can outweigh the time per core
        Default - False
    func_args : list, optional
        arguments to be passed to the function
    func_kwargs : dict, optional
        keyword arguments to be passed onto function
    joblib_backend : str, optional
        Backend to use for parallel computation with Joblib.
        The older paradigm - "multiprocessing" is the default in pyUSID.
        Set to None to use the joblib default - "loky"
    verbose : bool, optional. default = False
        Whether or not to print statements that aid in debugging

    Returns
    -------
    results : list
        List of computational results
    """ # Was unable to get the MPI + joblib framework to work. # Did not compute anything at all. Just froze # Finished reading the entire data set # List comprehension vs map vs for loop? # https://stackoverflow.com/questions/1247486/python-list-comprehension-vs-map
data
func
cores
lengthy_computation
func_args
func_kwargs
verbose
joblib_backend