timeseries_scripts/read.py
def read_opsd(filepath, url, headers):
    '''Read a file from OPSD into a DataFrame'''
    df = pd.read_csv(
        filepath,
        sep=',',
        header=0,
        index_col='timestamp',
        parse_dates={'timestamp': ['day']},
        date_parser=None,
        dayfirst=False,
        decimal='.',
        thousands=None,
        converters=None,
    )

    # The capacities data only has one entry per day, which pandas
    # interprets as 00:00h. We will broadcast the dayly data for
    # all quarter-hours of the day until the next given data point.
    # For this, we we expand the index so it reaches to 23:59 of
    # the last day, not only 00:00.
    last = pd.to_datetime([df.index[-1].replace(hour=23, minute=59)])
    until_last = df.index.append(last).rename('timestamp')
    df = df.reindex(index=until_last, method='ffill')
    df.index = df.index.tz_localize('Europe/Berlin')
    df.index = df.index.tz_convert(None)
    df = df.resample('15min').ffill()

    colmap = {
        'Solar': {
            'variable': 'solar',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        },
        'Onshore': {
            'variable': 'wind_onshore',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        },
        'Offshore': {
            'variable': 'wind_offshore',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        }
    }

    # Drop any column not in colmap
    df = df[list(colmap.keys())]

    # Create the MultiIndex
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)

    return df
Read a file from OPSD into a DataFrame
Read a file from OPSD into a DataFrame
Read a file from OPSD into a DataFrame
def read_opsd(filepath, url, headers):
    df = pd.read_csv(
        filepath,
        sep=',',
        header=0,
        index_col='timestamp',
        parse_dates={'timestamp': ['day']},
        date_parser=None,
        dayfirst=False,
        decimal='.',
        thousands=None,
        converters=None,
    )
    last = pd.to_datetime([df.index[-1].replace(hour=23, minute=59)])
    until_last = df.index.append(last).rename('timestamp')
    df = df.reindex(index=until_last, method='ffill')
    df.index = df.index.tz_localize('Europe/Berlin')
    df.index = df.index.tz_convert(None)
    df = df.resample('15min').ffill()
    colmap = {
        'Solar': {
            'variable': 'solar',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        },
        'Onshore': {
            'variable': 'wind_onshore',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        },
        'Offshore': {
            'variable': 'wind_offshore',
            'region': 'DE',
            'attribute': 'capacity',
            'source': 'BNetzA and Netztransparenz.de',
            'web': url,
            'unit': 'MW'
        }
    }
    df = df[list(colmap.keys())]
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_opsd ( filepath , url , headers ) : df = pd . read_csv ( filepath , sep = ',' , header = 0 , index_col = 'timestamp' , parse_dates = { 'timestamp' : [ 'day' ] } , date_parser = None , dayfirst = False , decimal = '.' , thousands = None , converters = None , ) last = pd . to_datetime ( [ df . index [ - 1 ] . replace ( hour = 23 , minute = 59 ) ] ) until_last = df . index . append ( last ) . rename ( 'timestamp' ) df = df . reindex ( index = until_last , method = 'ffill' ) df . index = df . index . tz_localize ( 'Europe/Berlin' ) df . index = df . index . tz_convert ( None ) df = df . resample ( '15min' ) . ffill ( ) colmap = { 'Solar' : { 'variable' : 'solar' , 'region' : 'DE' , 'attribute' : 'capacity' , 'source' : 'BNetzA and Netztransparenz.de' , 'web' : url , 'unit' : 'MW' } , 'Onshore' : { 'variable' : 'wind_onshore' , 'region' : 'DE' , 'attribute' : 'capacity' , 'source' : 'BNetzA and Netztransparenz.de' , 'web' : url , 'unit' : 'MW' } , 'Offshore' : { 'variable' : 'wind_offshore' , 'region' : 'DE' , 'attribute' : 'capacity' , 'source' : 'BNetzA and Netztransparenz.de' , 'web' : url , 'unit' : 'MW' } } df = df [ list ( colmap . keys ( ) ) ] tuples = [ tuple ( colmap [ col ] [ level ] for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a file from OPSD into a DataFrame
Read a file from OPSD into a DataFrame
'''Read a file from OPSD into a DataFrame''' # The capacities data only has one entry per day, which pandas # interprets as 00:00h. We will broadcast the dayly data for # all quarter-hours of the day until the next given data point. # For this, we we expand the index so it reaches to 23:59 of # the last day, not only 00:00. # Drop any column not in colmap # Create the MultiIndex
filepath
url
headers