timeseries_scripts/read.py
def read_rte(filepath, variable_name, url, headers):
    '''Read a file from RTE into a DataFrame'''
    #open zip
    myzipfile = zipfile.ZipFile(filepath, mode='r')
    myzipfile.extractall(path=(os.path.split(filepath)[0]))    
    
    #change path from zip to excel
    from os import walk
    f = []
    for (dirpath, dirnames, filenames) in walk((os.path.split(filepath)[0])):
        f.extend(filenames)
        break
    
    get_excel = [item for item in f if item.endswith('.xls')]
    
    filepath = os.path.join((os.path.split(filepath)[0]), get_excel[0])
            
    #open excel which is actually tsv
    with open(filepath, 'r') as f:        
        reader = csv.reader(f, delimiter='\t', lineterminator='\n')
        df = pd.DataFrame(list(reader))
        
    #delete the excel as read() throws exception if there are two files in one download directory
    #this means multiple runthroughs of this script would otherwise not be possible
    os.remove(filepath)
    
    df = df.iloc[:,:13] #make sure there are no empty columns at the end    
    df.columns = df.iloc[1] #set column names
    df['Heures'] = df['Heures'].map(lambda x: str(x).lstrip('Données de réalisation du ')) #strip the cells with dates of any other text
    df['Dates'] = np.nan
    
    #fill an extra column with corresponding dates.
    for f in df['Heures']:
        if re.match('\d{2}/\d{2}/\d{4}', f):
            df['Dates'][list(df['Heures']).index(f)]=f
    df['Dates'].fillna(method='ffill', axis=0, inplace=True, limit=25)
    
    #drop all rows not containing data (no time-format in first column)
    df = df.loc[df['Heures'].str.len()==11]
    
    #drop daylight saving time hours (no data there)
    df = df.loc[(df['Éolien terrestre'] != '*') & (df['Solaire'] != '*')]
    
    #just display beginning of hours
    df['Heures'] = df['Heures'].map(lambda x: str(x)[:-6])
    
    #construct full date to later use as index
    df['timestamp'] = df['Dates']+' '+df['Heures']
    df['timestamp'] = pd.to_datetime(df['timestamp'], dayfirst=True,)
    
    #drop autumn dst hours as they contain inconsistent data (none or copy of hour before)
    dst_transitions_autumn = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Paris')._utc_transition_times
        if d.year >= 2000 and d.month == 10]
    df = df.loc[~df['timestamp'].isin(dst_transitions_autumn)]
    df.set_index(df['timestamp'], inplace=True)
    
    #Transfer to UTC
    df.index = df.index.tz_localize('Europe/Paris')
    df.index = df.index.tz_convert(None)
    
    colmap = {
        'Éolien terrestre': {
            'variable': 'wind',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url
        },
        'Solaire': {
            'variable': 'solar',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url
        }
    }

    
    # Drop any column not in colmap
    df = df[list(colmap.keys())]
    
    # Create the MultiIndex    
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    
    return df
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
def read_rte(filepath, variable_name, url, headers):
    myzipfile = zipfile.ZipFile(filepath, mode='r')
    myzipfile.extractall(path=(os.path.split(filepath)[0]))    
    from os import walk
    f = []
    for (dirpath, dirnames, filenames) in walk((os.path.split(filepath)[0])):
        f.extend(filenames)
        break
    get_excel = [item for item in f if item.endswith('.xls')]
    filepath = os.path.join((os.path.split(filepath)[0]), get_excel[0])
    with open(filepath, 'r') as f:        
        reader = csv.reader(f, delimiter='\t', lineterminator='\n')
        df = pd.DataFrame(list(reader))
    os.remove(filepath)
    df = df.iloc[:,:13] 
    df.columns = df.iloc[1] 
    df['Heures'] = df['Heures'].map(lambda x: str(x).lstrip('Données de réalisation du ')) #s
    df['Dates'] = np.nan
    for f in df['Heures']:
        if re.match('\d{2}/\d{2}/\d{4}', f):
            df['Dates'][list(df['Heures']).index(f)]=f
    df['Dates'].fillna(method='ffill', axis=0, inplace=True, limit=25)
    df = df.loc[df['Heures'].str.len()==11]
    df = df.loc[(df['Éolien terrestre'] != '*') & (df['Solaire'] != '*')]
    df['Heures'] = df['Heures'].map(lambda x: str(x)[:-6])
    df['timestamp'] = df['Dates']+' '+df['Heures']
    df['timestamp'] = pd.to_datetime(df['timestamp'], dayfirst=True,)
    dst_transitions_autumn = [
        d.replace(hour=2)
        for d in pytz.timezone('Europe/Paris')._utc_transition_times
        if d.year >= 2000 and d.month == 10]
    df = df.loc[~df['timestamp'].isin(dst_transitions_autumn)]
    df.set_index(df['timestamp'], inplace=True)
    df.index = df.index.tz_localize('Europe/Paris')
    df.index = df.index.tz_convert(None)
    colmap = {
        'Éolien terrestre': {
            'variable': 'wind',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url
        },
        'Solaire': {
            'variable': 'solar',
            'region': 'FR',
            'attribute': 'generation',
            'source': 'RTE',
            'web': url
        }
    }
    df = df[list(colmap.keys())]
    tuples = [tuple(colmap[col][level] for level in headers)
              for col in df.columns]
    df.columns = pd.MultiIndex.from_tuples(tuples, names=headers)
    return df
def read_rte ( filepath , variable_name , url , headers ) : myzipfile = zipfile . ZipFile ( filepath , mode = 'r' ) myzipfile . extractall ( path = ( os . path . split ( filepath ) [ 0 ] ) ) from os import walk f = [ ] for ( dirpath , dirnames , filenames ) in walk ( ( os . path . split ( filepath ) [ 0 ] ) ) : f . extend ( filenames ) break get_excel = [ item for item in f if item . endswith ( '.xls' ) ] filepath = os . path . join ( ( os . path . split ( filepath ) [ 0 ] ) , get_excel [ 0 ] ) with open ( filepath , 'r' ) as f : reader = csv . reader ( f , delimiter = '\t' , lineterminator = '\n' ) df = pd . DataFrame ( list ( reader ) ) os . remove ( filepath ) df = df . iloc [ : , : 13 ] df . columns = df . iloc [ 1 ] df [ 'Heures' ] = df [ 'Heures' ] . map ( lambda x : str ( x ) . lstrip ( 'Données de réalisation du ')) # df [ 'Dates' ] = np . nan for f in df [ 'Heures' ] : if re . match ( '\d{2}/\d{2}/\d{4}' , f ) : df [ 'Dates' ] [ list ( df [ 'Heures' ] ) . index ( f ) ] = f df [ 'Dates' ] . fillna ( method = 'ffill' , axis = 0 , inplace = True , limit = 25 ) df = df . loc [ df [ 'Heures' ] . str . len ( ) == 11 ] df = df . loc [ ( df [ 'Éolien terrestre'] =  *') d f[ ' Solaire'] =  *') ] df [ 'Heures' ] = df [ 'Heures' ] . map ( lambda x : str ( x ) [ : - 6 ] ) df [ 'timestamp' ] = df [ 'Dates' ] + ' ' + df [ 'Heures' ] df [ 'timestamp' ] = pd . to_datetime ( df [ 'timestamp' ] , dayfirst = True , ) dst_transitions_autumn = [ d . replace ( hour = 2 ) for d in pytz . timezone ( 'Europe/Paris' ) . _utc_transition_times if d . year >= 2000 and d . month == 10 ] df = df . loc [ ~ df [ 'timestamp' ] . isin ( dst_transitions_autumn ) ] df . set_index ( df [ 'timestamp' ] , inplace = True ) df . index = df . index . tz_localize ( 'Europe/Paris' ) df . index = df . index . tz_convert ( None ) colmap = { 'Éolien terrestre': 'variable' : 'wind' , 'region' : 'FR' , 'attribute' : 'generation' , 'source' : 'RTE' , 'web' : url } , 'Solaire' : { 'variable' : 'solar' , 'region' : 'FR' , 'attribute' : 'generation' , 'source' : 'RTE' , 'web' : url } } df = df [ list ( colmap . keys ( ) ) ] tuples = [ tuple ( colmap [ col ] [ level ] for level in headers ) for col in df . columns ] df . columns = pd . MultiIndex . from_tuples ( tuples , names = headers ) return df
Read a file from RTE into a DataFrame
Read a file from RTE into a DataFrame
'''Read a file from RTE into a DataFrame''' #open zip #change path from zip to excel #open excel which is actually tsv #delete the excel as read() throws exception if there are two files in one download directory #this means multiple runthroughs of this script would otherwise not be possible #make sure there are no empty columns at the end     #set column names trip the cells with dates of any other text #fill an extra column with corresponding dates. #drop all rows not containing data (no time-format in first column) #drop daylight saving time hours (no data there) #just display beginning of hours #construct full date to later use as index #drop autumn dst hours as they contain inconsistent data (none or copy of hour before) #Transfer to UTC # Drop any column not in colmap # Create the MultiIndex    
filepath
variable_name
url
headers