timeseries_scripts/read.py
def read_pse(filepath, variable_name, url, headers):
    """
    Read a .csv file from PSE into a DataFrame.

    Parameters
    ----------
    filepath : str
        Directory path of file to be read
    variable_name : str
        Name of variable, e.g. ``solar``
    url : str
        URL linking to the source website where this data comes from
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe

    Returns
    ----------
    df: pandas.DataFrame
        The content of one file from PSE

    """

    df = pd.read_csv(
        filepath,
        sep=';',
        encoding='cp1250',
        header=0,
        index_col=None,
        parse_dates=None,
        date_parser=None,
        dayfirst=False,
        decimal=',',
        thousands=None,
        # hours are indicated by their ending time. During fall DST,
        # UTC 23:00-00:00 = CEST 1:00-2:00 is indicated by '02',
        # UTC 00:00-01:00 = CEST 2:00-3:00 is indicated by '02A',
        # UTC 01:00-02:00 = CET  2:00-3:00 is indicated by '03'.
        # regular hours require backshifting by 1 period
        converters={
            'Godzina':
                lambda x: '2:00' if x == '02A' else str(int(x) - 1) + ':00'
        }
    )
    # Create a list of spring-daylight savings time (DST)-transitions
    dst_transitions_spring = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Copenhagen')._utc_transition_times
        if d.year >= 2000 and d.month == 3]

    # The hour from 01:00 - 02:00 (CET) should by PSE's logic be indexed
    # by "02:00", but at DST day in spring they use "03:00" in the files.
    # Our routine requires it to be "01:00"
    df['proto_timestamp'] = pd.to_datetime(
        df['Data'].astype(str) + ' ' +
        df['Godzina'])
    slicer = df['proto_timestamp'].isin(dst_transitions_spring)
    df.loc[slicer, 'Godzina'] = '1:00'

    # create the actual timestamp from the corrected "Data"-column
    df['timestamp'] = pd.to_datetime(
        df['Data'].astype(str) + ' ' +
        df['Godzina'])
    df.set_index('timestamp', inplace=True)

    # 'ambigous' refers to how the October dst-transition hour is handled.
    # ‘infer’ will attempt to infer dst-transition hours based on order.
    df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    df.index = df.index.tz_convert(None)

    colmap = {
        'Sumaryczna generacja źródeł wiatrowych': {
            'region': 'PL',
            'variable': 'wind_onshore',
            'attribute': 'generation',
            'source': 'PSE',
            'web': url,
            'unit': 'MW'
        }
    }

    # Drop any column not in colmap
    df = df[list(colmap.keys())]

    # Create the MultiIndex
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)

    return df

    Read a .csv file from PSE into a DataFrame.

    Parameters
    ----------
    filepath : str
        Directory path of file to be read
    variable_name : str
        Name of variable, e.g. ``solar``
    url : str
        URL linking to the source website where this data comes from
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe

    Returns
    ----------
    df: pandas.DataFrame
        The content of one file from PSE

    
Read a .csv file from PSE into a DataFrame.
Parameters

filepath : str
Directory path of file to be read
variable_name : str
Name of variable, e.g. ``solar``
url : str
URL linking to the source website where this data comes from
headers : list
List of strings indicating the level names of the pandas.MultiIndex
for the columns of the dataframe

Returns

pandas.DataFrame
The content of one file from PSE
Read a . csv file from PSE into a DataFrame . Parameters filepath : str Directory path of file to be read variable_name : str Name of variable e . g . ` ` solar ` ` url : str URL linking to the source website where this data comes from headers : list List of strings indicating the level names of the pandas . MultiIndex for the columns of the dataframe Returns pandas . DataFrame The content of one file from PSE
def read_pse(filepath, variable_name, url, headers):
    df = pd.read_csv(
        filepath,
        sep=';',
        encoding='cp1250',
        header=0,
        index_col=None,
        parse_dates=None,
        date_parser=None,
        dayfirst=False,
        decimal=',',
        thousands=None,
        converters={
            'Godzina':
                lambda x: '2:00' if x == '02A' else str(int(x) - 1) + ':00'
        }
    )
    dst_transitions_spring = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Copenhagen')._utc_transition_times
        if d.year >= 2000 and d.month == 3]
    df['proto_timestamp'] = pd.to_datetime(
        df['Data'].astype(str) + ' ' +
        df['Godzina'])
    slicer = df['proto_timestamp'].isin(dst_transitions_spring)
    df.loc[slicer, 'Godzina'] = '1:00'
    df['timestamp'] = pd.to_datetime(
        df['Data'].astype(str) + ' ' +
        df['Godzina'])
    df.set_index('timestamp', inplace=True)
    df.index = df.index.tz_localize('Europe/Berlin', ambiguous='infer')
    df.index = df.index.tz_convert(None)
    colmap = {
        'Sumaryczna generacja źródeł wiatrowych': {
            'region': 'PL',
            'variable': 'wind_onshore',
            'attribute': 'generation',
            'source': 'PSE',
            'web': url,
            'unit': 'MW'
        }
    }
    df = df[list(colmap.keys())]
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_pse ( filepath , variable_name , url , headers ) : df = pd . read_csv ( filepath , sep = ';' , encoding = 'cp1250' , header = 0 , index_col = None , parse_dates = None , date_parser = None , dayfirst = False , decimal = ',' , thousands = None , converters = { 'Godzina' : lambda x : '2:00' if x == '02A' else str ( int ( x ) - 1 ) + ':00' } ) dst_transitions_spring = [ d . replace ( hour = 2 ) for d in pytz . timezone ( 'Europe/Copenhagen' ) . _utc_transition_times if d . year >= 2000 and d . month == 3 ] df [ 'proto_timestamp' ] = pd . to_datetime ( df [ 'Data' ] . astype ( str ) + ' ' + df [ 'Godzina' ] ) slicer = df [ 'proto_timestamp' ] . isin ( dst_transitions_spring ) df . loc [ slicer , 'Godzina' ] = '1:00' df [ 'timestamp' ] = pd . to_datetime ( df [ 'Data' ] . astype ( str ) + ' ' + df [ 'Godzina' ] ) df . set_index ( 'timestamp' , inplace = True ) df . index = df . index . tz_localize ( 'Europe/Berlin' , ambiguous = 'infer' ) df . index = df . index . tz_convert ( None ) colmap = { 'Sumaryczna generacja źródeł wiatrowych': { 'region' : 'PL' , 'variable' : 'wind_onshore' , 'attribute' : 'generation' , 'source' : 'PSE' , 'web' : url , 'unit' : 'MW' } } df = df [ list ( colmap . keys ( ) ) ] tuples = [ tuple ( colmap [ col ] [ level ] for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a .csv file from PSE into a DataFrame.
Read a . csv file from PSE into a DataFrame .
"""
    Read a .csv file from PSE into a DataFrame.

    Parameters
    ----------
    filepath : str
        Directory path of file to be read
    variable_name : str
        Name of variable, e.g. ``solar``
    url : str
        URL linking to the source website where this data comes from
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe

    Returns
    ----------
    df: pandas.DataFrame
        The content of one file from PSE

    """ # hours are indicated by their ending time. During fall DST, # UTC 23:00-00:00 = CEST 1:00-2:00 is indicated by '02', # UTC 00:00-01:00 = CEST 2:00-3:00 is indicated by '02A', # UTC 01:00-02:00 = CET  2:00-3:00 is indicated by '03'. # regular hours require backshifting by 1 period # Create a list of spring-daylight savings time (DST)-transitions # The hour from 01:00 - 02:00 (CET) should by PSE's logic be indexed # by "02:00", but at DST day in spring they use "03:00" in the files. # Our routine requires it to be "01:00" # create the actual timestamp from the corrected "Data"-column # 'ambigous' refers to how the October dst-transition hour is handled. # ‘infer’ will attempt to infer dst-transition hours based on order. # Drop any column not in colmap # Create the MultiIndex
filepath
variable_name
url
headers