timeseries_scripts/imputation.py
def impute(nan_region, col, col_name, nan_regs, df, one_period):
    '''
    Impute missing value spans longer than one hour based on other TSOs.

    Parameters
    ----------
    nan_region : pandas.Series
        Contains information on one region of missing data in col
    col : pandas.DataFrame
        A column from df as a separate DataFrame 
    col_name : tuple
        tuple of header levels of column to inspect
    nan_regs : : pandas.DataFrame
        DataFrame with each row representing a region of missing data in col
    df : pandas.DataFrame
        DataFrame to patch
    one_period : pandas.Timedelta
        Time resolution of df and col (15/60 minutes)

    '''
    #logger.info('guessed %s entries after %s', row['count'], row['start_idx'])
    day_before = pd.DatetimeIndex(
        freq='15min',
        start=nan_region['start_idx'] - timedelta(hours=24),
        end=nan_region['start_idx'] - one_period)

    to_fill = pd.DatetimeIndex(
        freq='15min',
        start=nan_region['start_idx'],
        end=nan_region['till_idx'])

    # other_tsos = [c[1] for c in compact.drop(col_name, axis=1)
    #.loc[:,(col_name[0],slice(None),col_name[2])].columns.tolist()]
    other_tsos = [tso
                  for tso in ['DE-50Hertz', 'DE-Amprion', 'DE-TenneT', 'DE-TransnetBW']
                  if tso != col_name[1]]

    # select columns with data for same technology (wind/solar) but from other
    # TSOs
    similar = df.loc[:, (col_name[0], other_tsos, col_name[2])]
    # calculate the sum using columns without NaNs the day
    # before or during the period to be guessed
    similar = similar.dropna(
        axis=1,
        how='any',
        subset=day_before.append(to_fill)
    ).sum(axis=1)
    # calculate scaling factor for other TSO data
    factor = similar.loc[day_before].sum(
        axis=0) / col.loc[day_before, :].sum(axis=0)

    guess = similar.loc[to_fill] / float(factor)
    col.iloc[:, 0].loc[to_fill] = guess
    a = float(col.iloc[:, 0].loc[nan_region['start_idx'] - one_period])
    b = float(col.iloc[:, 0].loc[nan_region['start_idx']])
    if a == 0:
        deviation = '{} absolut'.format(a - b)
    else:
        deviation = '{:.2f} %'.format((a - b) / a * 100)
    logger.info(
        '%s : \n        '
        'guessed %s entries after %s \n        '
        'last non-missing: %s \n        '
        'first guessed: %s \n        '
        'deviation of first guess from last known value: %s',
        col_name[0:3], nan_region['count'], nan_region[
            'start_idx'], a, b, deviation
    )

    return col

    Impute missing value spans longer than one hour based on other TSOs.

    Parameters
    ----------
    nan_region : pandas.Series
        Contains information on one region of missing data in col
    col : pandas.DataFrame
        A column from df as a separate DataFrame 
    col_name : tuple
        tuple of header levels of column to inspect
    nan_regs : : pandas.DataFrame
        DataFrame with each row representing a region of missing data in col
    df : pandas.DataFrame
        DataFrame to patch
    one_period : pandas.Timedelta
        Time resolution of df and col (15/60 minutes)

    
Impute missing value spans longer than one hour based on other TSOs.
Parameters


Impute missing value spans longer than one hour based on other TSOs . Parameters
def impute(nan_region, col, col_name, nan_regs, df, one_period):
    day_before = pd.DatetimeIndex(
        freq='15min',
        start=nan_region['start_idx'] - timedelta(hours=24),
        end=nan_region['start_idx'] - one_period)
    to_fill = pd.DatetimeIndex(
        freq='15min',
        start=nan_region['start_idx'],
        end=nan_region['till_idx'])
    other_tsos = [tso
                  for tso in ['DE-50Hertz', 'DE-Amprion', 'DE-TenneT', 'DE-TransnetBW']
                  if tso != col_name[1]]
    similar = df.loc[:, (col_name[0], other_tsos, col_name[2])]
    similar = similar.dropna(
        axis=1,
        how='any',
        subset=day_before.append(to_fill)
    ).sum(axis=1)
    factor = similar.loc[day_before].sum(
        axis=0) / col.loc[day_before, :].sum(axis=0)
    guess = similar.loc[to_fill] / float(factor)
    col.iloc[:, 0].loc[to_fill] = guess
    a = float(col.iloc[:, 0].loc[nan_region['start_idx'] - one_period])
    b = float(col.iloc[:, 0].loc[nan_region['start_idx']])
    if a == 0:
        deviation = '{} absolut'.format(a - b)
    else:
        deviation = '{:.2f} %'.format((a - b) / a * 100)
    logger.info(
        '%s : \n        '
        'guessed %s entries after %s \n        '
        'last non-missing: %s \n        '
        'first guessed: %s \n        '
        'deviation of first guess from last known value: %s',
        col_name[0:3], nan_region['count'], nan_region[
            'start_idx'], a, b, deviation
    )
    return col
def impute ( nan_region , col , col_name , nan_regs , df , one_period ) : day_before = pd . DatetimeIndex ( freq = '15min' , start = nan_region [ 'start_idx' ] - timedelta ( hours = 24 ) , end = nan_region [ 'start_idx' ] - one_period ) to_fill = pd . DatetimeIndex ( freq = '15min' , start = nan_region [ 'start_idx' ] , end = nan_region [ 'till_idx' ] ) other_tsos = [ tso for tso in [ 'DE-50Hertz' , 'DE-Amprion' , 'DE-TenneT' , 'DE-TransnetBW' ] if tso != col_name [ 1 ] ] similar = df . loc [ : , ( col_name [ 0 ] , other_tsos , col_name [ 2 ] ) ] similar = similar . dropna ( axis = 1 , how = 'any' , subset = day_before . append ( to_fill ) ) . sum ( axis = 1 ) factor = similar . loc [ day_before ] . sum ( axis = 0 ) / col . loc [ day_before , : ] . sum ( axis = 0 ) guess = similar . loc [ to_fill ] / float ( factor ) col . iloc [ : , 0 ] . loc [ to_fill ] = guess a = float ( col . iloc [ : , 0 ] . loc [ nan_region [ 'start_idx' ] - one_period ] ) b = float ( col . iloc [ : , 0 ] . loc [ nan_region [ 'start_idx' ] ] ) if a == 0 : deviation = '{} absolut' . format ( a - b ) else : deviation = '{:.2f} %' . format ( ( a - b ) / a * 100 ) logger . info ( '%s : \n        ' 'guessed %s entries after %s \n        ' 'last non-missing: %s \n        ' 'first guessed: %s \n        ' 'deviation of first guess from last known value: %s' , col_name [ 0 : 3 ] , nan_region [ 'count' ] , nan_region [ 'start_idx' ] , a , b , deviation ) return col
Impute missing value spans longer than one hour based on other TSOs.
Impute missing value spans longer than one hour based on other TSOs .
'''
    Impute missing value spans longer than one hour based on other TSOs.

    Parameters
    ----------
    nan_region : pandas.Series
        Contains information on one region of missing data in col
    col : pandas.DataFrame
        A column from df as a separate DataFrame 
    col_name : tuple
        tuple of header levels of column to inspect
    nan_regs : : pandas.DataFrame
        DataFrame with each row representing a region of missing data in col
    df : pandas.DataFrame
        DataFrame to patch
    one_period : pandas.Timedelta
        Time resolution of df and col (15/60 minutes)

    ''' #logger.info('guessed %s entries after %s', row['count'], row['start_idx']) # other_tsos = [c[1] for c in compact.drop(col_name, axis=1) #.loc[:,(col_name[0],slice(None),col_name[2])].columns.tolist()] # select columns with data for same technology (wind/solar) but from other # TSOs # calculate the sum using columns without NaNs the day # before or during the period to be guessed # calculate scaling factor for other TSO data
nan_region
col
col_name
nan_regs
df
one_period