timeseries_scripts/read.py
def read_rte(filepath, variable_name, url, headers):
    '''Read a file from RTE into a DataFrame'''

    # pandas.read_csv infers the table dimensions from the header row.
    # Since the first row uses only one column, it needs to be read separately
    # in order not to mess up the DataFrame
    df1 = pd.read_csv(
        filepath,
        sep='\t',
        encoding='cp1252',
        compression='zip',
        nrows=1,
        header=None
    )
    df2 = pd.read_csv(
        filepath,
        sep='\t',
        encoding='cp1252',
        compression='zip',
        skiprows=[0],
        header=None,
        usecols=list(range(0, 13))
    )

    # Glue the DataFrames together
    df = pd.concat([df1, df2], ignore_index=True)

    # set column names
    df = df.rename(columns=df.iloc[1])

    # strip the cells with dates of any other text
    df['Heures'] = df['Heures'].str.lstrip('Données de réalisation du ')

    # fill an extra column with corresponding dates.
    df['Dates'] = np.nan
    slicer = df['Heures'].str.match('\d{2}/\d{2}/\d{4}')
    df['Dates'] = df['Heures'].loc[slicer]
    df['Dates'].fillna(method='ffill', axis=0, inplace=True, limit=25)

    # drop all rows not containing data (no time-format in first column)
    df = df.loc[df['Heures'].str.len() == 11]

    # drop daylight saving time hours (no data there)
    df = df.loc[(df['Éolien terrestre'] != '*') & (df['Solaire'] != '*')]

    # just display beginning of hours
    df['Heures'] = df['Heures'].str[:5]

    # construct full date to later use as index
    df['timestamp'] = df['Dates'] + ' ' + df['Heures']
    df['timestamp'] = pd.to_datetime(df['timestamp'], dayfirst=True,)

    # drop autumn dst hours as they contain inconsistent data (none or copy of
    # hour before)
    dst_transitions_autumn = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Paris')._utc_transition_times
        if d.year >= 2000 and d.month == 10]
    df = df.loc[~df['timestamp'].isin(dst_transitions_autumn)]
    df.set_index(df['timestamp'], inplace=True)

    # Transfer to UTC
    df.index = df.index.tz_localize('Europe/Paris')
    df.index = df.index.tz_convert(None)

    colmap = {
        'Éolien terrestre': {
            'variable': 'wind_onshore',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url,
            'unit': 'MW'
        },
        'Solaire': {
            'variable': 'solar',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url,
            'unit': 'MW'
        }
    }

    # Drop any column not in colmap
    df = df[[key for key in colmap.keys() if key in df.columns]]

    # Create the MultiIndex
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)

    return df
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
def read_rte(filepath, variable_name, url, headers):
    df1 = pd.read_csv(
        filepath,
        sep='\t',
        encoding='cp1252',
        compression='zip',
        nrows=1,
        header=None
    )
    df2 = pd.read_csv(
        filepath,
        sep='\t',
        encoding='cp1252',
        compression='zip',
        skiprows=[0],
        header=None,
        usecols=list(range(0, 13))
    )
    df = pd.concat([df1, df2], ignore_index=True)
    df = df.rename(columns=df.iloc[1])
    df['Heures'] = df['Heures'].str.lstrip('Données de réalisation du ')
    df['Dates'] = np.nan
    slicer = df['Heures'].str.match('\d{2}/\d{2}/\d{4}')
    df['Dates'] = df['Heures'].loc[slicer]
    df['Dates'].fillna(method='ffill', axis=0, inplace=True, limit=25)
    df = df.loc[df['Heures'].str.len() == 11]
    df = df.loc[(df['Éolien terrestre'] != '*') & (df['Solaire'] != '*')]
    df['Heures'] = df['Heures'].str[:5]
    df['timestamp'] = df['Dates'] + ' ' + df['Heures']
    df['timestamp'] = pd.to_datetime(df['timestamp'], dayfirst=True,)
    dst_transitions_autumn = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Paris')._utc_transition_times
        if d.year >= 2000 and d.month == 10]
    df = df.loc[~df['timestamp'].isin(dst_transitions_autumn)]
    df.set_index(df['timestamp'], inplace=True)
    df.index = df.index.tz_localize('Europe/Paris')
    df.index = df.index.tz_convert(None)
    colmap = {
        'Éolien terrestre': {
            'variable': 'wind_onshore',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url,
            'unit': 'MW'
        },
        'Solaire': {
            'variable': 'solar',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url,
            'unit': 'MW'
        }
    }
    df = df[[key for key in colmap.keys() if key in df.columns]]
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_rte ( filepath , variable_name , url , headers ) : df1 = pd . read_csv ( filepath , sep = '\t' , encoding = 'cp1252' , compression = 'zip' , nrows = 1 , header = None ) df2 = pd . read_csv ( filepath , sep = '\t' , encoding = 'cp1252' , compression = 'zip' , skiprows = [ 0 ] , header = None , usecols = list ( range ( 0 , 13 ) ) ) df = pd . concat ( [ df1 , df2 ] , ignore_index = True ) df = df . rename ( columns = df . iloc [ 1 ] ) df [ 'Heures' ] = df [ 'Heures' ] . str . lstrip ( 'Données de réalisation du ') df [ 'Dates' ] = np . nan slicer = df [ 'Heures' ] . str . match ( '\d{2}/\d{2}/\d{4}' ) df [ 'Dates' ] = df [ 'Heures' ] . loc [ slicer ] df [ 'Dates' ] . fillna ( method = 'ffill' , axis = 0 , inplace = True , limit = 25 ) df = df . loc [ df [ 'Heures' ] . str . len ( ) == 11 ] df = df . loc [ ( df [ 'Éolien terrestre'] =  *') d f[ ' Solaire'] =  *') ] df [ 'Heures' ] = df [ 'Heures' ] . str [ : 5 ] df [ 'timestamp' ] = df [ 'Dates' ] + ' ' + df [ 'Heures' ] df [ 'timestamp' ] = pd . to_datetime ( df [ 'timestamp' ] , dayfirst = True , ) dst_transitions_autumn = [ d . replace ( hour = 2 ) for d in pytz . timezone ( 'Europe/Paris' ) . _utc_transition_times if d . year >= 2000 and d . month == 10 ] df = df . loc [ ~ df [ 'timestamp' ] . isin ( dst_transitions_autumn ) ] df . set_index ( df [ 'timestamp' ] , inplace = True ) df . index = df . index . tz_localize ( 'Europe/Paris' ) df . index = df . index . tz_convert ( None ) colmap = { 'Éolien terrestre': 'variable' : 'wind_onshore' , 'region' : 'FR' , 'attribute' : 'generation' , 'source' : 'RTE' , 'web' : url , 'unit' : 'MW' } , 'Solaire' : { 'variable' : 'solar' , 'region' : 'FR' , 'attribute' : 'generation' , 'source' : 'RTE' , 'web' : url , 'unit' : 'MW' } } df = df [ [ key for key in colmap . keys ( ) if key in df . columns ] ] tuples = [ tuple ( colmap [ col ] [ level ] for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
'''Read a file from RTE into a DataFrame''' # pandas.read_csv infers the table dimensions from the header row. # Since the first row uses only one column, it needs to be read separately # in order not to mess up the DataFrame # Glue the DataFrames together # set column names # strip the cells with dates of any other text # fill an extra column with corresponding dates. # drop all rows not containing data (no time-format in first column) # drop daylight saving time hours (no data there) # just display beginning of hours # construct full date to later use as index # drop autumn dst hours as they contain inconsistent data (none or copy of # hour before) # Transfer to UTC # Drop any column not in colmap # Create the MultiIndex
filepath
variable_name
url
headers