timeseries_scripts/read.py
def read_hertz(filepath, variable_name, url, headers):
    '''Read a file from 50Hertz into a DataFrame'''
    df = pd.read_csv(
        filepath,
        sep=';',
        header=3,
        index_col='timestamp',
        parse_dates={'timestamp': ['Datum', 'Von']},
        date_parser=None,
        dayfirst=True,
        decimal=',',
        thousands='.',
        # truncate values in 'time' column after 5th character
        converters={'Von': lambda x: x[:5]},
    )

    # Until 2006, and in 2015 (except for wind_generation_pre-offshore),
    # during the fall dst-transistion, only the
    # wintertime hour (marked by a B in the data) is reported, the summertime
    # hour, (marked by an A) is missing in the data.
    # dst_arr is a boolean array consisting only of "False" entries, telling
    # python to treat the hour from 2:00 to 2:59 as wintertime.
    if (pd.to_datetime(df.index.values[0]).year not in [2005, 2006, 2015] or
            (variable_name == 'wind_generation_pre-offshore' and
             pd.to_datetime(df.index.values[0]).year == 2015)):
        df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    else:
        dst_arr = np.zeros(len(df.index), dtype=bool)
        df.index = df.index.tz_localize('Europe/Berlin', ambiguous=dst_arr)

    df.index = df.index.tz_convert(None)

    tech, attribute = variable_name.split('_')[:2]

    colmap = {
        'MW': {
            'variable': '{tech}',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        },
        'Onshore MW': {
            'variable': 'wind_onshore',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        },
        'Offshore MW': {
            'variable': 'wind_offshore',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        }
    }
    # Since 2016, wind data has an aditional column for offshore.
    # Baltic 1 has been producing since 2011-05-02 and Baltic2 since
    # early 2015 (source: Wikipedia) so it is probably not correct that
    # 50Hertz-Wind data pre-2016 is only onshore. Maybe we can ask at
    # 50Hertz directly.

    # Drop any column not in colmap
    df = df[[key for key in colmap.keys() if key in df.columns]]

    # Create the MultiIndex
    tuples = [tuple(colmap[col][level].format(tech=tech, attribute=attribute)
                    for level in headers) for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)

    return df
Read a file from 50Hertz into a DataFrame
Read a file from 50Hertz into a DataFrame
Read a file from 50Hertz into a DataFrame
def read_hertz(filepath, variable_name, url, headers):
    df = pd.read_csv(
        filepath,
        sep=';',
        header=3,
        index_col='timestamp',
        parse_dates={'timestamp': ['Datum', 'Von']},
        date_parser=None,
        dayfirst=True,
        decimal=',',
        thousands='.',
        converters={'Von': lambda x: x[:5]},
    )
    if (pd.to_datetime(df.index.values[0]).year not in [2005, 2006, 2015] or
            (variable_name == 'wind_generation_pre-offshore' and
             pd.to_datetime(df.index.values[0]).year == 2015)):
        df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    else:
        dst_arr = np.zeros(len(df.index), dtype=bool)
        df.index = df.index.tz_localize('Europe/Berlin', ambiguous=dst_arr)
    df.index = df.index.tz_convert(None)
    tech, attribute = variable_name.split('_')[:2]
    colmap = {
        'MW': {
            'variable': '{tech}',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        },
        'Onshore MW': {
            'variable': 'wind_onshore',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        },
        'Offshore MW': {
            'variable': 'wind_offshore',
            'region': 'DE_50hertz',
            'attribute': '{attribute}',
            'source': '50Hertz',
            'web': url,
            'unit': 'MW'
        }
    }
    df = df[[key for key in colmap.keys() if key in df.columns]]
    tuples = [tuple(colmap[col][level].format(tech=tech, attribute=attribute)
                    for level in headers) for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_hertz ( filepath , variable_name , url , headers ) : df = pd . read_csv ( filepath , sep = ';' , header = 3 , index_col = 'timestamp' , parse_dates = { 'timestamp' : [ 'Datum' , 'Von' ] } , date_parser = None , dayfirst = True , decimal = ',' , thousands = '.' , converters = { 'Von' : lambda x : x [ : 5 ] } , ) if ( pd . to_datetime ( df . index . values [ 0 ] ) . year not in [ 2005 , 2006 , 2015 ] or ( variable_name == 'wind_generation_pre-offshore' and pd . to_datetime ( df . index . values [ 0 ] ) . year == 2015 ) ) : df . index = df . index . tz_localize ( 'Europe/Berlin' , ambiguous = 'infer' ) else : dst_arr = np . zeros ( len ( df . index ) , dtype = bool ) df . index = df . index . tz_localize ( 'Europe/Berlin' , ambiguous = dst_arr ) df . index = df . index . tz_convert ( None ) tech , attribute = variable_name . split ( '_' ) [ : 2 ] colmap = { 'MW' : { 'variable' : '{tech}' , 'region' : 'DE_50hertz' , 'attribute' : '{attribute}' , 'source' : '50Hertz' , 'web' : url , 'unit' : 'MW' } , 'Onshore MW' : { 'variable' : 'wind_onshore' , 'region' : 'DE_50hertz' , 'attribute' : '{attribute}' , 'source' : '50Hertz' , 'web' : url , 'unit' : 'MW' } , 'Offshore MW' : { 'variable' : 'wind_offshore' , 'region' : 'DE_50hertz' , 'attribute' : '{attribute}' , 'source' : '50Hertz' , 'web' : url , 'unit' : 'MW' } } df = df [ [ key for key in colmap . keys ( ) if key in df . columns ] ] tuples = [ tuple ( colmap [ col ] [ level ] . format ( tech = tech , attribute = attribute ) for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a file from 50Hertz into a DataFrame
Read a file from 50Hertz into a DataFrame
'''Read a file from 50Hertz into a DataFrame''' # truncate values in 'time' column after 5th character # Until 2006, and in 2015 (except for wind_generation_pre-offshore), # during the fall dst-transistion, only the # wintertime hour (marked by a B in the data) is reported, the summertime # hour, (marked by an A) is missing in the data. # dst_arr is a boolean array consisting only of "False" entries, telling # python to treat the hour from 2:00 to 2:59 as wintertime. # Since 2016, wind data has an aditional column for offshore. # Baltic 1 has been producing since 2011-05-02 and Baltic2 since # early 2015 (source: Wikipedia) so it is probably not correct that # 50Hertz-Wind data pre-2016 is only onshore. Maybe we can ask at # 50Hertz directly. # Drop any column not in colmap # Create the MultiIndex
filepath
variable_name
url
headers