parallelpandas/mp_impl.py
def groupby_apply(dataframe, columns, func,
                  n_processes=mp.cpu_count(), mp_ctx=None):
    """
    parallel version of groupby/apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    columns : list or Index
        subset of columns to used to group by
    func : function
        the function pass to apply of the grouper object
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all

    Returns
    -------
    result of groupby/apply operation
    """
    # Pickle function
    f_pickled = dill.dumps(func)
    # Segmentize rows such that two rows belonging to the same group
    # are assigned to the same segment
    segments = apply(dataframe[columns],
                     lambda x: hash(tuple(x.values)) % n_processes,
                     n_processes=mp.cpu_count(), axis=1)
    # Init and start workers
    result_queue = mp_ctx.Queue()
    processes = []
    for iSegment in range(n_processes):
        p = mp_ctx.Process(target=groupby_apply_worker,
                       args=(result_queue, dataframe, columns,
                             f_pickled, iSegment, segments))
        p.daemon = True
        p.start()
        processes.append(p)
    # Get results
    results = list()
    while len(results) < n_processes:
        results.append(result_queue.get())
    # Wait for all workers to finish
    for p in processes:
        p.join()
    # Close queues
    result_queue.close()
    # Return concated results
    return pd.concat(results)

    parallel version of groupby/apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    columns : list or Index
        subset of columns to used to group by
    func : function
        the function pass to apply of the grouper object
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all

    Returns
    -------
    result of groupby/apply operation
    
parallel version of groupby/apply
Parameters

data : Series or DataFrame
the pandas object to apply data on
columns : list or Index
subset of columns to used to group by
func : function
the function pass to apply of the grouper object
n_processes : int (optional)
specify the number of processes to use, otherwise use all

Returns

result of groupby/apply operation
parallel version of groupby / apply Parameters data : Series or DataFrame the pandas object to apply data on columns : list or Index subset of columns to used to group by func : function the function pass to apply of the grouper object n_processes : int ( optional ) specify the number of processes to use otherwise use all Returns result of groupby / apply operation
def groupby_apply(dataframe, columns, func,
                  n_processes=mp.cpu_count(), mp_ctx=None):
    f_pickled = dill.dumps(func)
    segments = apply(dataframe[columns],
                     lambda x: hash(tuple(x.values)) % n_processes,
                     n_processes=mp.cpu_count(), axis=1)
    result_queue = mp_ctx.Queue()
    processes = []
    for iSegment in range(n_processes):
        p = mp_ctx.Process(target=groupby_apply_worker,
                       args=(result_queue, dataframe, columns,
                             f_pickled, iSegment, segments))
        p.daemon = True
        p.start()
        processes.append(p)
    results = list()
    while len(results) < n_processes:
        results.append(result_queue.get())
    for p in processes:
        p.join()
    result_queue.close()
    return pd.concat(results)
def groupby_apply ( dataframe , columns , func , n_processes = mp . cpu_count ( ) , mp_ctx = None ) : f_pickled = dill . dumps ( func ) segments = apply ( dataframe [ columns ] , lambda x : hash ( tuple ( x . values ) ) % n_processes , n_processes = mp . cpu_count ( ) , axis = 1 ) result_queue = mp_ctx . Queue ( ) processes = [ ] for iSegment in range ( n_processes ) : p = mp_ctx . Process ( target = groupby_apply_worker , args = ( result_queue , dataframe , columns , f_pickled , iSegment , segments ) ) p . daemon = True p . start ( ) processes . append ( p ) results = list ( ) while len ( results ) < n_processes : results . append ( result_queue . get ( ) ) for p in processes : p . join ( ) result_queue . close ( ) return pd . concat ( results )
parallel version of groupby/apply
Parameters
parallel version of groupby / apply Parameters
"""
    parallel version of groupby/apply

    Parameters
    ----------
    data : Series or DataFrame
        the pandas object to apply data on
    columns : list or Index
        subset of columns to used to group by
    func : function
        the function pass to apply of the grouper object
    n_processes : int (optional)
        specify the number of processes to use, otherwise use all

    Returns
    -------
    result of groupby/apply operation
    """ # Pickle function # Segmentize rows such that two rows belonging to the same group # are assigned to the same segment # Init and start workers # Get results # Wait for all workers to finish # Close queues # Return concated results
dataframe
columns
func
n_processes
mp_ctx