parallelpandas/mp_impl.py
def apply(data, f, n_processes=mp.cpu_count(), mp_ctx=None, **kwargs):
    """
    parallel version of apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    f : function
        the function to apply
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all
    Additional keyword arguments will be passed on to pandas apply operation

    Returns
    -------
    result of apply operation
    """
    # Compute chunk size
    chunk_size = (len(data)-1) // n_processes + 1
    # Pickle function
    f_pickled = dill.dumps(f)
    # Start workers
    result_queue = mp_ctx.Queue()
    processes = []
    for offset in range(n_processes):
        p = mp_ctx.Process(target=apply_worker,
                       args=(result_queue, data, offset, chunk_size,
                             f_pickled, kwargs))
        p.daemon = True
        p.start()
        processes.append(p)
    # Get results
    results = list()
    while len(results) < n_processes:
        results.append(result_queue.get())
    # Wait for all workers to finish
    for p in processes:
        p.join()
    # Close queues
    result_queue.close()
    # Sort results such that resulting series/dataframe has the same index
    # as the original one
    results = [y[1] for y in sorted(results, key=lambda x: x[0])]
    # Return concated results
    return pd.concat(results)

    parallel version of apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    f : function
        the function to apply
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all
    Additional keyword arguments will be passed on to pandas apply operation

    Returns
    -------
    result of apply operation
    
parallel version of apply
Parameters

data : Series or DataFrame
the pandas object to apply data on
f : function
the function to apply
n_processes : int (optional)
specify the number of processes to use, otherwise use all
Additional keyword arguments will be passed on to pandas apply operation

Returns

result of apply operation
parallel version of apply Parameters data : Series or DataFrame the pandas object to apply data on f : function the function to apply n_processes : int ( optional ) specify the number of processes to use otherwise use all Additional keyword arguments will be passed on to pandas apply operation Returns result of apply operation
def apply(data, f, n_processes=mp.cpu_count(), mp_ctx=None, **kwargs):
    chunk_size = (len(data)-1) // n_processes + 1
    f_pickled = dill.dumps(f)
    result_queue = mp_ctx.Queue()
    processes = []
    for offset in range(n_processes):
        p = mp_ctx.Process(target=apply_worker,
                       args=(result_queue, data, offset, chunk_size,
                             f_pickled, kwargs))
        p.daemon = True
        p.start()
        processes.append(p)
    results = list()
    while len(results) < n_processes:
        results.append(result_queue.get())
    for p in processes:
        p.join()
    result_queue.close()
    results = [y[1] for y in sorted(results, key=lambda x: x[0])]
    return pd.concat(results)
def apply ( data , f , n_processes = mp . cpu_count ( ) , mp_ctx = None , ** kwargs ) : chunk_size = ( len ( data ) - 1 ) // n_processes + 1 f_pickled = dill . dumps ( f ) result_queue = mp_ctx . Queue ( ) processes = [ ] for offset in range ( n_processes ) : p = mp_ctx . Process ( target = apply_worker , args = ( result_queue , data , offset , chunk_size , f_pickled , kwargs ) ) p . daemon = True p . start ( ) processes . append ( p ) results = list ( ) while len ( results ) < n_processes : results . append ( result_queue . get ( ) ) for p in processes : p . join ( ) result_queue . close ( ) results = [ y [ 1 ] for y in sorted ( results , key = lambda x : x [ 0 ] ) ] return pd . concat ( results )
parallel version of apply
Parameters
parallel version of apply Parameters
"""
    parallel version of apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    f : function
        the function to apply
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all
    Additional keyword arguments will be passed on to pandas apply operation

    Returns
    -------
    result of apply operation
    """ # Compute chunk size # Pickle function # Start workers # Get results # Wait for all workers to finish # Close queues # Sort results such that resulting series/dataframe has the same index # as the original one # Return concated results
data
f
n_processes
mp_ctx