timeseries_scripts/read.py
def read_tennet(filepath, variable_name, url, headers):
    '''Read a file from TenneT into a DataFrame'''
    df = pd.read_csv(
        filepath,
        sep=';',
        encoding='latin_1',
        header=3,
        index_col=False,
        parse_dates=False,
        date_parser=None,
        dayfirst=True,
        thousands=None,
        converters=None,
    )

    renamer = {'Datum': 'date', 'Position': 'pos'}
    df = df.rename(columns=renamer)

    df['date'].fillna(method='ffill', limit=100, inplace=True)

    # Check the rows for irregularities
    for i in range(len(df.index)):
        # On the day in March when summertime begins, shift the data forward by
        # 1 hour, beginning with the 9th quarter-hour, so the index runs again
        # up to 96
        if (df['pos'][i] == 92 and ((i == len(df.index) - 1) or
                                    (df['pos'][i + 1] == 1))):
            slicer = df[(df['date'] == df['date'][i]) & (df['pos'] >= 9)].index
            df.loc[slicer, 'pos'] = df['pos'] + 4

        elif df['pos'][i] > 96:  # True when summertime ends in October
            logger.debug('%s th quarter-hour at %s, position %s',
                         df['pos'][i], df.ix[i, 'date'], (i))

            # Instead of having the quarter-hours' index run up to 100, we want
            # to have it set back by 1 hour beginning from the 13th
            # quarter-hour, ending at 96
            if df['pos'][i] == 100 and not (df['pos'] == 101).any():
                slicer = df[(df['date'] == df['date'][i])
                            & (df['pos'] >= 13)].index
                df.loc[slicer, 'pos'] = df['pos'] - 4

    # Compute timestamp from position and generate datetime-index
    df['hour'] = (np.trunc((df['pos'] - 1) / 4)).astype(int).astype(str)
    df['minute'] = (((df['pos'] - 1) % 4) * 15).astype(int).astype(str)
    df['timestamp'] = pd.to_datetime(df['date'] + ' ' + df['hour'] + ':' +
                                     df['minute'], dayfirst=True)
    df.set_index('timestamp', inplace=True)

    df.drop(['pos', 'date', 'hour', 'minute'], axis=1, inplace=True)

    df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    df.index = df.index.tz_convert(None)

    tech = variable_name
    colmap = {
        'prognostiziert [MW]': {
            'variable': '{tech}',
            'region': 'DE_tennet',
            'attribute': 'forecast',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        },
        'tatsächlich [MW]': {
            'variable': '{tech}',
            'region': 'DE_tennet',
            'attribute': 'generation',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        },
        'Anteil Offshore [MW]': {
            'variable': 'wind_offshore',
            'region': 'DE_tennet',
            'attribute': 'generation',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        }
    }

    # Drop any column not in colmap
    df = df[[key for key in colmap.keys() if key in df.columns]]

    # Create the MultiIndex
    tuples = [tuple(colmap[col][level].format(tech=tech) for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)

    return df
Read a file from TenneT into a DataFrame
Read a file from TenneT into a DataFrame
Read a file from TenneT into a DataFrame
def read_tennet(filepath, variable_name, url, headers):
    df = pd.read_csv(
        filepath,
        sep=';',
        encoding='latin_1',
        header=3,
        index_col=False,
        parse_dates=False,
        date_parser=None,
        dayfirst=True,
        thousands=None,
        converters=None,
    )
    renamer = {'Datum': 'date', 'Position': 'pos'}
    df = df.rename(columns=renamer)
    df['date'].fillna(method='ffill', limit=100, inplace=True)
    for i in range(len(df.index)):
        if (df['pos'][i] == 92 and ((i == len(df.index) - 1) or
                                    (df['pos'][i + 1] == 1))):
            slicer = df[(df['date'] == df['date'][i]) & (df['pos'] >= 9)].index
            df.loc[slicer, 'pos'] = df['pos'] + 4
        elif df['pos'][i] > 96:  
            logger.debug('%s th quarter-hour at %s, position %s',
                         df['pos'][i], df.ix[i, 'date'], (i))
            if df['pos'][i] == 100 and not (df['pos'] == 101).any():
                slicer = df[(df['date'] == df['date'][i])
                            & (df['pos'] >= 13)].index
                df.loc[slicer, 'pos'] = df['pos'] - 4
    df['hour'] = (np.trunc((df['pos'] - 1) / 4)).astype(int).astype(str)
    df['minute'] = (((df['pos'] - 1) % 4) * 15).astype(int).astype(str)
    df['timestamp'] = pd.to_datetime(df['date'] + ' ' + df['hour'] + ':' +
                                     df['minute'], dayfirst=True)
    df.set_index('timestamp', inplace=True)
    df.drop(['pos', 'date', 'hour', 'minute'], axis=1, inplace=True)
    df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    df.index = df.index.tz_convert(None)
    tech = variable_name
    colmap = {
        'prognostiziert [MW]': {
            'variable': '{tech}',
            'region': 'DE_tennet',
            'attribute': 'forecast',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        },
        'tatsächlich [MW]': {
            'variable': '{tech}',
            'region': 'DE_tennet',
            'attribute': 'generation',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        },
        'Anteil Offshore [MW]': {
            'variable': 'wind_offshore',
            'region': 'DE_tennet',
            'attribute': 'generation',
            'source': 'TenneT',
            'web': url,
            'unit': 'MW'
        }
    }
    df = df[[key for key in colmap.keys() if key in df.columns]]
    tuples = [tuple(colmap[col][level].format(tech=tech) for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_tennet ( filepath , variable_name , url , headers ) : df = pd . read_csv ( filepath , sep = ';' , encoding = 'latin_1' , header = 3 , index_col = False , parse_dates = False , date_parser = None , dayfirst = True , thousands = None , converters = None , ) renamer = { 'Datum' : 'date' , 'Position' : 'pos' } df = df . rename ( columns = renamer ) df [ 'date' ] . fillna ( method = 'ffill' , limit = 100 , inplace = True ) for i in range ( len ( df . index ) ) : if ( df [ 'pos' ] [ i ] == 92 and ( ( i == len ( df . index ) - 1 ) or ( df [ 'pos' ] [ i + 1 ] == 1 ) ) ) : slicer = df [ ( df [ 'date' ] == df [ 'date' ] [ i ] ) & ( df [ 'pos' ] >= 9 ) ] . index df . loc [ slicer , 'pos' ] = df [ 'pos' ] + 4 elif df [ 'pos' ] [ i ] > 96 : logger . debug ( '%s th quarter-hour at %s, position %s' , df [ 'pos' ] [ i ] , df . ix [ i , 'date' ] , ( i ) ) if df [ 'pos' ] [ i ] == 100 and not ( df [ 'pos' ] == 101 ) . any ( ) : slicer = df [ ( df [ 'date' ] == df [ 'date' ] [ i ] ) & ( df [ 'pos' ] >= 13 ) ] . index df . loc [ slicer , 'pos' ] = df [ 'pos' ] - 4 df [ 'hour' ] = ( np . trunc ( ( df [ 'pos' ] - 1 ) / 4 ) ) . astype ( int ) . astype ( str ) df [ 'minute' ] = ( ( ( df [ 'pos' ] - 1 ) % 4 ) * 15 ) . astype ( int ) . astype ( str ) df [ 'timestamp' ] = pd . to_datetime ( df [ 'date' ] + ' ' + df [ 'hour' ] + ':' + df [ 'minute' ] , dayfirst = True ) df . set_index ( 'timestamp' , inplace = True ) df . drop ( [ 'pos' , 'date' , 'hour' , 'minute' ] , axis = 1 , inplace = True ) df . index = df . index . tz_localize ( 'Europe/Berlin' , ambiguous = 'infer' ) df . index = df . index . tz_convert ( None ) tech = variable_name colmap = { 'prognostiziert [MW]' : { 'variable' : '{tech}' , 'region' : 'DE_tennet' , 'attribute' : 'forecast' , 'source' : 'TenneT' , 'web' : url , 'unit' : 'MW' } , 'tatsächlich [MW]': 'variable' : '{tech}' , 'region' : 'DE_tennet' , 'attribute' : 'generation' , 'source' : 'TenneT' , 'web' : url , 'unit' : 'MW' } , 'Anteil Offshore [MW]' : { 'variable' : 'wind_offshore' , 'region' : 'DE_tennet' , 'attribute' : 'generation' , 'source' : 'TenneT' , 'web' : url , 'unit' : 'MW' } } df = df [ [ key for key in colmap . keys ( ) if key in df . columns ] ] tuples = [ tuple ( colmap [ col ] [ level ] . format ( tech = tech ) for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a file from TenneT into a DataFrame
Read a file from TenneT into a DataFrame
'''Read a file from TenneT into a DataFrame''' # Check the rows for irregularities # On the day in March when summertime begins, shift the data forward by # 1 hour, beginning with the 9th quarter-hour, so the index runs again # up to 96 # True when summertime ends in October # Instead of having the quarter-hours' index run up to 100, we want # to have it set back by 1 hour beginning from the 13th # quarter-hour, ending at 96 # Compute timestamp from position and generate datetime-index # Drop any column not in colmap # Create the MultiIndex
filepath
variable_name
url
headers