timeseries_scripts/read.py
def read(data_path, areas, source_name, variable_name, url, res_key,
         headers, start_from_user=None, end_from_user=None):
    """
    For the sources specified in the sources.yml file, pass each downloaded
    file to the correct read function.

    Parameters
    ----------
    source_name : str
        Name of source to read files from
    variable_name : str
        Indicator for subset of data available together in the same files
    url : str
        URL of the Source to be placed in the column-MultiIndex
    res_key : str
        Resolution of the source data. Must be one of ['15min', '60min']
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    data_path : str, default: 'original_data'
        Base download directory in which to save all downloaded files
    start_from_user : datetime.date, default None
        Start of period for which to read the data
    end_from_user : datetime.date, default None
        End of period for which to read the data

    Returns
    ----------
    data_set: pandas.DataFrame
        A DataFrame containing the combined data for variable_name 

    """
    data_set = pd.DataFrame()

    variable_dir = os.path.join(data_path, source_name, variable_name)

    logger.info('reading %s - %s', source_name, variable_name)

    files_existing = sum([len(files) for r, d, files in os.walk(variable_dir)])
    files_success = 0

    # Check if there are folders for variable_name
    if not os.path.exists(variable_dir):
        logger.warning('folder not found for %s, %s',
                       source_name, variable_name)
        return data_set

    # For each file downloaded for that variable
    for container in sorted(os.listdir(variable_dir)):
        # Skip this file if period covered excluded by user
        if start_from_user:
            # Filecontent is too old
            if start_from_user > yaml.load(container.split('_')[1]):
                continue  # go to next container

        if end_from_user:
            # Filecontent is too recent
            if end_from_user < yaml.load(container.split('_')[0]):
                continue  # go to next container

        files = os.listdir(os.path.join(variable_dir, container))

        # Check if there is only one file per folder
        if len(files) == 0:
            logger.warning('found no file in %s %s %s',
                           source_name, variable_name, container)
            continue

        elif len(files) > 1:
            logger.warning('found more than one file in %s %s %s',
                           source_name, variable_name, container)
            continue

        filepath = os.path.join(variable_dir, container, files[0])

        # Check if file is not empty
        if os.path.getsize(filepath) < 128:
            logger.warning('%s \n file is smaller than 128 Byte. It is probably'
                           ' empty and will thus be skipped from reading',
                           filepath)
        else:
            logger.debug('reading data:\n\t '
                         'Source:   %s\n\t '
                         'Variable: %s\n\t '
                         'Filename: %s',
                         source_name, variable_name, files[0])

            update_progress(files_success, files_existing)

            if source_name == 'OPSD':
                data_to_add = read_opsd(filepath, url, headers)
            elif source_name == 'CEPS':
                data_to_add = read_ceps(filepath, variable_name, url, headers)
            elif source_name == 'ENTSO-E Transparency FTP':
                data_to_add = read_entso_e_transparency(
                    areas, filepath, variable_name, url, headers, res_key)
            elif source_name == 'ENTSO-E Data Portal':
                #save_stdout = sys.stdout
                #sys.stdout = open('trash', 'w')
                data_to_add = read_entso_e_portal(filepath, url, headers)
                #sys.stdout = save_stdout
            elif source_name == 'Energinet.dk':
                data_to_add = read_energinet_dk(filepath, url, headers)
            elif source_name == 'Elia':
                data_to_add = read_elia(filepath, variable_name, url, headers)
            elif source_name == 'PSE':
                data_to_add = read_pse(filepath, variable_name, url, headers)
            elif source_name == 'RTE':
                data_to_add = read_rte(filepath, variable_name, url, headers)
            elif source_name == 'Svenska Kraftnaet':
                data_to_add = read_svenska_kraftnaet(
                    filepath, variable_name, url, headers)
            elif source_name == '50Hertz':
                data_to_add = read_hertz(filepath, variable_name, url, headers)
            elif source_name == 'Amprion':
                data_to_add = read_amprion(
                    filepath, variable_name, url, headers)
            elif source_name == 'TenneT':
                data_to_add = read_tennet(
                    filepath, variable_name, url, headers)
            elif source_name == 'TransnetBW':
                data_to_add = read_transnetbw(
                    filepath, variable_name, url, headers)
            elif source_name == 'APG':
                data_to_add = read_apg(filepath, url, headers)

            if data_set.empty:
                data_set = data_to_add
            else:
                data_set = data_set.combine_first(data_to_add)

            files_success += 1
            update_progress(files_success, files_existing)

    if data_set.empty:
        logger.warning('returned empty DataFrame for %s, %s',
                       source_name, variable_name)
        return data_set

    # Reindex with a new index that is sure to be continous in order to later
    # expose gaps in the data.
    no_gaps = pd.DatetimeIndex(start=data_set.index[0],
                               end=data_set.index[-1],
                               freq=res_key)
    data_set = data_set.reindex(index=no_gaps)

    # Cut off the data outside of [start_from_user:end_from_user]
    # First, convert userinput to UTC time to conform with data_set.index
    if start_from_user:
        start_from_user = (
            pytz.timezone('Europe/Brussels')
            .localize(datetime.combine(start_from_user, time()))
            .astimezone(pytz.timezone('UTC')))
    if end_from_user:
        end_from_user = (
            pytz.timezone('Europe/Brussels')
            .localize(datetime.combine(end_from_user, time()))
            .astimezone(pytz.timezone('UTC'))
            # appropriate offset to inlude the end of period
            + timedelta(days=1, minutes=-int(res_key[:2])))
    # Then cut off the data_set
    data_set = data_set.loc[start_from_user:end_from_user, :]

    return data_set

    For the sources specified in the sources.yml file, pass each downloaded
    file to the correct read function.

    Parameters
    ----------
    source_name : str
        Name of source to read files from
    variable_name : str
        Indicator for subset of data available together in the same files
    url : str
        URL of the Source to be placed in the column-MultiIndex
    res_key : str
        Resolution of the source data. Must be one of ['15min', '60min']
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    data_path : str, default: 'original_data'
        Base download directory in which to save all downloaded files
    start_from_user : datetime.date, default None
        Start of period for which to read the data
    end_from_user : datetime.date, default None
        End of period for which to read the data

    Returns
    ----------
    data_set: pandas.DataFrame
        A DataFrame containing the combined data for variable_name 

    
For the sources specified in the sources.yml file, pass each downloaded
file to the correct read function.

Parameters

source_name : str
Name of source to read files from
variable_name : str
Indicator for subset of data available together in the same files
url : str
URL of the Source to be placed in the column-MultiIndex
res_key : str
Resolution of the source data.

Returns


For the sources specified in the sources . yml file pass each downloaded file to the correct read function . Parameters source_name : str Name of source to read files from variable_name : str Indicator for subset of data available together in the same files url : str URL of the Source to be placed in the column - MultiIndex res_key : str Resolution of the source data . Returns
def read(data_path, areas, source_name, variable_name, url, res_key,
         headers, start_from_user=None, end_from_user=None):
    data_set = pd.DataFrame()
    variable_dir = os.path.join(data_path, source_name, variable_name)
    logger.info('reading %s - %s', source_name, variable_name)
    files_existing = sum([len(files) for r, d, files in os.walk(variable_dir)])
    files_success = 0
    if not os.path.exists(variable_dir):
        logger.warning('folder not found for %s, %s',
                       source_name, variable_name)
        return data_set
    for container in sorted(os.listdir(variable_dir)):
        if start_from_user:
            if start_from_user > yaml.load(container.split('_')[1]):
                continue  
        if end_from_user:
            if end_from_user < yaml.load(container.split('_')[0]):
                continue  
        files = os.listdir(os.path.join(variable_dir, container))
        if len(files) == 0:
            logger.warning('found no file in %s %s %s',
                           source_name, variable_name, container)
            continue
        elif len(files) > 1:
            logger.warning('found more than one file in %s %s %s',
                           source_name, variable_name, container)
            continue
        filepath = os.path.join(variable_dir, container, files[0])
        if os.path.getsize(filepath) < 128:
            logger.warning('%s \n file is smaller than 128 Byte. It is probably'
                           ' empty and will thus be skipped from reading',
                           filepath)
        else:
            logger.debug('reading data:\n\t '
                         'Source:   %s\n\t '
                         'Variable: %s\n\t '
                         'Filename: %s',
                         source_name, variable_name, files[0])
            update_progress(files_success, files_existing)
            if source_name == 'OPSD':
                data_to_add = read_opsd(filepath, url, headers)
            elif source_name == 'CEPS':
                data_to_add = read_ceps(filepath, variable_name, url, headers)
            elif source_name == 'ENTSO-E Transparency FTP':
                data_to_add = read_entso_e_transparency(
                    areas, filepath, variable_name, url, headers, res_key)
            elif source_name == 'ENTSO-E Data Portal':
                data_to_add = read_entso_e_portal(filepath, url, headers)
            elif source_name == 'Energinet.dk':
                data_to_add = read_energinet_dk(filepath, url, headers)
            elif source_name == 'Elia':
                data_to_add = read_elia(filepath, variable_name, url, headers)
            elif source_name == 'PSE':
                data_to_add = read_pse(filepath, variable_name, url, headers)
            elif source_name == 'RTE':
                data_to_add = read_rte(filepath, variable_name, url, headers)
            elif source_name == 'Svenska Kraftnaet':
                data_to_add = read_svenska_kraftnaet(
                    filepath, variable_name, url, headers)
            elif source_name == '50Hertz':
                data_to_add = read_hertz(filepath, variable_name, url, headers)
            elif source_name == 'Amprion':
                data_to_add = read_amprion(
                    filepath, variable_name, url, headers)
            elif source_name == 'TenneT':
                data_to_add = read_tennet(
                    filepath, variable_name, url, headers)
            elif source_name == 'TransnetBW':
                data_to_add = read_transnetbw(
                    filepath, variable_name, url, headers)
            elif source_name == 'APG':
                data_to_add = read_apg(filepath, url, headers)
            if data_set.empty:
                data_set = data_to_add
            else:
                data_set = data_set.combine_first(data_to_add)
            files_success += 1
            update_progress(files_success, files_existing)
    if data_set.empty:
        logger.warning('returned empty DataFrame for %s, %s',
                       source_name, variable_name)
        return data_set
    no_gaps = pd.DatetimeIndex(start=data_set.index[0],
                               end=data_set.index[-1],
                               freq=res_key)
    data_set = data_set.reindex(index=no_gaps)
    if start_from_user:
        start_from_user = (
            pytz.timezone('Europe/Brussels')
            .localize(datetime.combine(start_from_user, time()))
            .astimezone(pytz.timezone('UTC')))
    if end_from_user:
        end_from_user = (
            pytz.timezone('Europe/Brussels')
            .localize(datetime.combine(end_from_user, time()))
            .astimezone(pytz.timezone('UTC'))
            + timedelta(days=1, minutes=-int(res_key[:2])))
    data_set = data_set.loc[start_from_user:end_from_user, :]
    return data_set
def read ( data_path , areas , source_name , variable_name , url , res_key , headers , start_from_user = None , end_from_user = None ) : data_set = pd . DataFrame ( ) variable_dir = os . path . join ( data_path , source_name , variable_name ) logger . info ( 'reading %s - %s' , source_name , variable_name ) files_existing = sum ( [ len ( files ) for r , d , files in os . walk ( variable_dir ) ] ) files_success = 0 if not os . path . exists ( variable_dir ) : logger . warning ( 'folder not found for %s, %s' , source_name , variable_name ) return data_set for container in sorted ( os . listdir ( variable_dir ) ) : if start_from_user : if start_from_user > yaml . load ( container . split ( '_' ) [ 1 ] ) : continue if end_from_user : if end_from_user < yaml . load ( container . split ( '_' ) [ 0 ] ) : continue files = os . listdir ( os . path . join ( variable_dir , container ) ) if len ( files ) == 0 : logger . warning ( 'found no file in %s %s %s' , source_name , variable_name , container ) continue elif len ( files ) > 1 : logger . warning ( 'found more than one file in %s %s %s' , source_name , variable_name , container ) continue filepath = os . path . join ( variable_dir , container , files [ 0 ] ) if os . path . getsize ( filepath ) < 128 : logger . warning ( '%s \n file is smaller than 128 Byte. It is probably' ' empty and will thus be skipped from reading' , filepath ) else : logger . debug ( 'reading data:\n\t ' 'Source:   %s\n\t ' 'Variable: %s\n\t ' 'Filename: %s' , source_name , variable_name , files [ 0 ] ) update_progress ( files_success , files_existing ) if source_name == 'OPSD' : data_to_add = read_opsd ( filepath , url , headers ) elif source_name == 'CEPS' : data_to_add = read_ceps ( filepath , variable_name , url , headers ) elif source_name == 'ENTSO-E Transparency FTP' : data_to_add = read_entso_e_transparency ( areas , filepath , variable_name , url , headers , res_key ) elif source_name == 'ENTSO-E Data Portal' : data_to_add = read_entso_e_portal ( filepath , url , headers ) elif source_name == 'Energinet.dk' : data_to_add = read_energinet_dk ( filepath , url , headers ) elif source_name == 'Elia' : data_to_add = read_elia ( filepath , variable_name , url , headers ) elif source_name == 'PSE' : data_to_add = read_pse ( filepath , variable_name , url , headers ) elif source_name == 'RTE' : data_to_add = read_rte ( filepath , variable_name , url , headers ) elif source_name == 'Svenska Kraftnaet' : data_to_add = read_svenska_kraftnaet ( filepath , variable_name , url , headers ) elif source_name == '50Hertz' : data_to_add = read_hertz ( filepath , variable_name , url , headers ) elif source_name == 'Amprion' : data_to_add = read_amprion ( filepath , variable_name , url , headers ) elif source_name == 'TenneT' : data_to_add = read_tennet ( filepath , variable_name , url , headers ) elif source_name == 'TransnetBW' : data_to_add = read_transnetbw ( filepath , variable_name , url , headers ) elif source_name == 'APG' : data_to_add = read_apg ( filepath , url , headers ) if data_set . empty : data_set = data_to_add else : data_set = data_set . combine_first ( data_to_add ) files_success += 1 update_progress ( files_success , files_existing ) if data_set . empty : logger . warning ( 'returned empty DataFrame for %s, %s' , source_name , variable_name ) return data_set no_gaps = pd . DatetimeIndex ( start = data_set . index [ 0 ] , end = data_set . index [ - 1 ] , freq = res_key ) data_set = data_set . reindex ( index = no_gaps ) if start_from_user : start_from_user = ( pytz . timezone ( 'Europe/Brussels' ) . localize ( datetime . combine ( start_from_user , time ( ) ) ) . astimezone ( pytz . timezone ( 'UTC' ) ) ) if end_from_user : end_from_user = ( pytz . timezone ( 'Europe/Brussels' ) . localize ( datetime . combine ( end_from_user , time ( ) ) ) . astimezone ( pytz . timezone ( 'UTC' ) ) + timedelta ( days = 1 , minutes = - int ( res_key [ : 2 ] ) ) ) data_set = data_set . loc [ start_from_user : end_from_user , : ] return data_set
For the sources specified in the sources.yml file, pass each downloaded
file to the correct read function.
For the sources specified in the sources . yml file pass each downloaded file to the correct read function .
"""
    For the sources specified in the sources.yml file, pass each downloaded
    file to the correct read function.

    Parameters
    ----------
    source_name : str
        Name of source to read files from
    variable_name : str
        Indicator for subset of data available together in the same files
    url : str
        URL of the Source to be placed in the column-MultiIndex
    res_key : str
        Resolution of the source data. Must be one of ['15min', '60min']
    headers : list
        List of strings indicating the level names of the pandas.MultiIndex
        for the columns of the dataframe
    data_path : str, default: 'original_data'
        Base download directory in which to save all downloaded files
    start_from_user : datetime.date, default None
        Start of period for which to read the data
    end_from_user : datetime.date, default None
        End of period for which to read the data

    Returns
    ----------
    data_set: pandas.DataFrame
        A DataFrame containing the combined data for variable_name 

    """ # Check if there are folders for variable_name # For each file downloaded for that variable # Skip this file if period covered excluded by user # Filecontent is too old # go to next container # Filecontent is too recent # go to next container # Check if there is only one file per folder # Check if file is not empty #save_stdout = sys.stdout #sys.stdout = open('trash', 'w') #sys.stdout = save_stdout # Reindex with a new index that is sure to be continous in order to later # expose gaps in the data. # Cut off the data outside of [start_from_user:end_from_user] # First, convert userinput to UTC time to conform with data_set.index # appropriate offset to inlude the end of period # Then cut off the data_set
data_path
areas
source_name
variable_name
url
res_key
headers
start_from_user
end_from_user