timeseries_scripts/imputation.py
def find_nan(df, res_key, headers, patch=False):
    '''
    Search for missing values in a DataFrame and optionally apply further 
    functions on each column.

    Parameters
    ----------    
    df : pandas.DataFrame
        DataFrame to inspect and possibly patch
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    patch : bool, default=False
        If False, return unaltered DataFrame,
        if True, return patched DataFrame

    Returns
    ----------    
    patched: pandas.DataFrame
        original df or df with gaps patched and marker column appended
    nan_table: pandas.DataFrame
        Contains detailed information about missing data

    '''
    nan_table = pd.DataFrame()
    patched = pd.DataFrame()
    marker_col = pd.Series(np.nan, index=df.index)

    if df.empty:
        return patched, nan_table

    # Get the frequency/length of one period of df
    one_period = pd.Timedelta(res_key)
    for col_name, col in df.iteritems():
        col = col.to_frame()
        message = '| {:5.5} | {:6.6} | {:10.10} | {:10.10} | {:10.10} | '.format(
            res_key, *col_name[0:4])

        # make an empty list of NaN regions to use as default
        nan_idx = pd.MultiIndex.from_arrays([
            [0, 0, 0, 0],
            ['count', 'span', 'start_idx', 'till_idx']])
        nan_list = pd.DataFrame(index=nan_idx, columns=col.columns)

        # skip this column if it has no entries at all.
        # This will also delete the column from the patched df
        if col.empty:
            continue

        # tag all occurences of NaN in the data with True
        # (but not before first or after last actual entry)
        col['tag'] = (
            (col.index >= col.first_valid_index()) &
            (col.index <= col.last_valid_index()) &
            col.isnull().transpose().as_matrix()
        ).transpose()

        # make another DF to hold info about each region
        nan_regs = pd.DataFrame()

        # first row of consecutive region is a True preceded by a False in tags
        nan_regs['start_idx'] = col.index[col['tag'] & ~
                                          col['tag'].shift(1).fillna(False)]

        # last row of consecutive region is a False preceded by a True
        nan_regs['till_idx'] = col.index[
            col['tag'] & ~
            col['tag'].shift(-1).fillna(False)]

        # if there are no NaNs, do nothing
        if not col['tag'].any():
            logger.info(message + 'nothing to patch in this column')
            col.drop('tag', axis=1, inplace=True)

        # else make a list of the NaN regions
        else:
            # how long is each region
            nan_regs['span'] = (
                nan_regs['till_idx'] - nan_regs['start_idx'] + one_period)
            nan_regs['count'] = (nan_regs['span'] / one_period)
            # sort the nan_regs DataFtame to put longest missing region on top
            nan_regs = nan_regs.sort_values(
                'count', ascending=False).reset_index(drop=True)

            col.drop('tag', axis=1, inplace=True)
            nan_list = nan_regs.stack().to_frame()
            nan_list.columns = col.columns

            if patch:
                col, marker_col = choose_fill_method(
                    message,
                    col, col_name, nan_regs, df, marker_col, one_period)

        if patched.empty:
            patched = col
        else:
            patched = patched.combine_first(col)

        if nan_table.empty:
            nan_table = nan_list
        else:
            nan_table = nan_table.combine_first(nan_list)

    # append the marker to the DataFrame
    marker_col = marker_col.to_frame()
    tuples = [('interpolated_values', '', '', '', '', '')]
    marker_col.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    patched = pd.concat([patched, marker_col], axis=1)

    # set the level names for the output
    nan_table.columns.names = headers
    patched.columns.names = headers

    return patched, nan_table

    Search for missing values in a DataFrame and optionally apply further 
    functions on each column.

    Parameters
    ----------    
    df : pandas.DataFrame
        DataFrame to inspect and possibly patch
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    patch : bool, default=False
        If False, return unaltered DataFrame,
        if True, return patched DataFrame

    Returns
    ----------    
    patched: pandas.DataFrame
        original df or df with gaps patched and marker column appended
    nan_table: pandas.DataFrame
        Contains detailed information about missing data

    
Search for missing values in a DataFrame and optionally apply further
functions on each column.
Search for missing values in a DataFrame and optionally apply further functions on each column .
def find_nan(df, res_key, headers, patch=False):
    nan_table = pd.DataFrame()
    patched = pd.DataFrame()
    marker_col = pd.Series(np.nan, index=df.index)
    if df.empty:
        return patched, nan_table
    one_period = pd.Timedelta(res_key)
    for col_name, col in df.iteritems():
        col = col.to_frame()
        message = '| {:5.5} | {:6.6} | {:10.10} | {:10.10} | {:10.10} | '.format(
            res_key, *col_name[0:4])
        nan_idx = pd.MultiIndex.from_arrays([
            [0, 0, 0, 0],
            ['count', 'span', 'start_idx', 'till_idx']])
        nan_list = pd.DataFrame(index=nan_idx, columns=col.columns)
        if col.empty:
            continue
        col['tag'] = (
            (col.index >= col.first_valid_index()) &
            (col.index <= col.last_valid_index()) &
            col.isnull().transpose().as_matrix()
        ).transpose()
        nan_regs = pd.DataFrame()
        nan_regs['start_idx'] = col.index[col['tag'] & ~
                                          col['tag'].shift(1).fillna(False)]
        nan_regs['till_idx'] = col.index[
            col['tag'] & ~
            col['tag'].shift(-1).fillna(False)]
        if not col['tag'].any():
            logger.info(message + 'nothing to patch in this column')
            col.drop('tag', axis=1, inplace=True)
        else:
            nan_regs['span'] = (
                nan_regs['till_idx'] - nan_regs['start_idx'] + one_period)
            nan_regs['count'] = (nan_regs['span'] / one_period)
            nan_regs = nan_regs.sort_values(
                'count', ascending=False).reset_index(drop=True)
            col.drop('tag', axis=1, inplace=True)
            nan_list = nan_regs.stack().to_frame()
            nan_list.columns = col.columns
            if patch:
                col, marker_col = choose_fill_method(
                    message,
                    col, col_name, nan_regs, df, marker_col, one_period)
        if patched.empty:
            patched = col
        else:
            patched = patched.combine_first(col)
        if nan_table.empty:
            nan_table = nan_list
        else:
            nan_table = nan_table.combine_first(nan_list)
    marker_col = marker_col.to_frame()
    tuples = [('interpolated_values', '', '', '', '', '')]
    marker_col.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    patched = pd.concat([patched, marker_col], axis=1)
    nan_table.columns.names = headers
    patched.columns.names = headers
    return patched, nan_table
def find_nan ( df , res_key , headers , patch = False ) : nan_table = pd . DataFrame ( ) patched = pd . DataFrame ( ) marker_col = pd . Series ( np . nan , index = df . index ) if df . empty : return patched , nan_table one_period = pd . Timedelta ( res_key ) for col_name , col in df . iteritems ( ) : col = col . to_frame ( ) message = '| {:5.5} | {:6.6} | {:10.10} | {:10.10} | {:10.10} | ' . format ( res_key , * col_name [ 0 : 4 ] ) nan_idx = pd . MultiIndex . from_arrays ( [ [ 0 , 0 , 0 , 0 ] , [ 'count' , 'span' , 'start_idx' , 'till_idx' ] ] ) nan_list = pd . DataFrame ( index = nan_idx , columns = col . columns ) if col . empty : continue col [ 'tag' ] = ( ( col . index >= col . first_valid_index ( ) ) & ( col . index <= col . last_valid_index ( ) ) & col . isnull ( ) . transpose ( ) . as_matrix ( ) ) . transpose ( ) nan_regs = pd . DataFrame ( ) nan_regs [ 'start_idx' ] = col . index [ col [ 'tag' ] & ~ col [ 'tag' ] . shift ( 1 ) . fillna ( False ) ] nan_regs [ 'till_idx' ] = col . index [ col [ 'tag' ] & ~ col [ 'tag' ] . shift ( - 1 ) . fillna ( False ) ] if not col [ 'tag' ] . any ( ) : logger . info ( message + 'nothing to patch in this column' ) col . drop ( 'tag' , axis = 1 , inplace = True ) else : nan_regs [ 'span' ] = ( nan_regs [ 'till_idx' ] - nan_regs [ 'start_idx' ] + one_period ) nan_regs [ 'count' ] = ( nan_regs [ 'span' ] / one_period ) nan_regs = nan_regs . sort_values ( 'count' , ascending = False ) . reset_index ( drop = True ) col . drop ( 'tag' , axis = 1 , inplace = True ) nan_list = nan_regs . stack ( ) . to_frame ( ) nan_list . columns = col . columns if patch : col , marker_col = choose_fill_method ( message , col , col_name , nan_regs , df , marker_col , one_period ) if patched . empty : patched = col else : patched = patched . combine_first ( col ) if nan_table . empty : nan_table = nan_list else : nan_table = nan_table . combine_first ( nan_list ) marker_col = marker_col . to_frame ( ) tuples = [ ( 'interpolated_values' , '' , '' , '' , '' , '' ) ] marker_col . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) patched = pd . concat ( [ patched , marker_col ] , axis = 1 ) nan_table . columns . names = headers patched . columns . names = headers return patched , nan_table
Search for missing values in a DataFrame and optionally apply further
functions on each column.
Search for missing values in a DataFrame and optionally apply further functions on each column .
'''
    Search for missing values in a DataFrame and optionally apply further 
    functions on each column.

    Parameters
    ----------    
    df : pandas.DataFrame
        DataFrame to inspect and possibly patch
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    patch : bool, default=False
        If False, return unaltered DataFrame,
        if True, return patched DataFrame

    Returns
    ----------    
    patched: pandas.DataFrame
        original df or df with gaps patched and marker column appended
    nan_table: pandas.DataFrame
        Contains detailed information about missing data

    ''' # Get the frequency/length of one period of df # make an empty list of NaN regions to use as default # skip this column if it has no entries at all. # This will also delete the column from the patched df # tag all occurences of NaN in the data with True # (but not before first or after last actual entry) # make another DF to hold info about each region # first row of consecutive region is a True preceded by a False in tags # last row of consecutive region is a False preceded by a True # if there are no NaNs, do nothing # else make a list of the NaN regions # how long is each region # sort the nan_regs DataFtame to put longest missing region on top # append the marker to the DataFrame # set the level names for the output
df
DataFrame to inspect and possibly patch
DataFrame to inspect and possibly patch
res_key
headers
List of strings indicating the level names of the pandas.MultiIndex
for the columns of the dataframe
List of strings indicating the level names of the pandas . MultiIndex for the columns of the dataframe
patch
If False, return unaltered DataFrame,
if True, return patched DataFrame
If False return unaltered DataFrame if True return patched DataFrame
Returns
None
----------    
None
patched
original df or df with gaps patched and marker column appended
original df or df with gaps patched and marker column appended
nan_table
Contains detailed information about missing data
Contains detailed information about missing data