timeseries_scripts/download.py
def download_request(
        source_name,
        start,
        end,
        session,
        filename,
        container,
        url_template,
        url_params_template,
        second=None):
    """
    Download a single file via HTTP get.
    Build the url from parameters and save the file to dsik under it's original
    filename 

    Parameters
    ----------
    container : str
        unique filepath for the file to be saved
    url_template : 
        stem of URL 
    url_params_template : dict
        dict of parameter names and values to paste into URL

    Returns
    ----------
    downloaded : bool
        True if download successful, False otherwise.
    session : requests.session

    """
    url_params = {}  # A dict for URL-parameters

    # For most sources, we can use HTTP get method with parameters-dict
    if url_params_template:
        for key, value in url_params_template.items():
            url_params[key] = value.format(
                u_start=start,
                u_end=end,
                u_second=second
            )
        url = url_template

    # For other sources that use urls without parameters (e.g. Svenska
    # Kraftnaet)
    else:
        url = url_template.format(
            u_start=start,
            u_end=end,
            u_second=second
        )

    resp = session.get(url, params=url_params)
    # For polish TSO PSE, URLs have been guessed.
    # Don't proceed for wrong guesses
    if source_name == 'PSE':
        if resp.text in ['Brak uprawnieñ', 'Brak uprawnień']:
            downloaded = False
            return downloaded, session

    # Get the original filename
    try:
        original_filename = (
            resp.headers['content-disposition']
            .split('filename=')[-1]
            .replace('"', '')
            .replace(';', '')
        )
        logger.debug('Downloaded from URL: %s Original filename: %s',
                     resp.url, original_filename)

    # For cases where the original filename can not be retrieved,
    # I put the filename in the param_dict
    except KeyError:
        if filename:
            original_filename = filename.format(u_start=start, u_end=end)
        else:
            logger.info(
                'original filename could neither be retrieved from server '
                'nor sources.yml'
            )
            original_filename = 'unknown_filename'

        logger.debug('Downloaded from URL: %s', resp.url)

    # Save file to disk
    filepath = os.path.join(container, original_filename)
    with open(filepath, 'wb') as output_file:
        for chunk in resp.iter_content(1024):
            output_file.write(chunk)
    downloaded = True
    return downloaded, session

    Download a single file via HTTP get.
    Build the url from parameters and save the file to dsik under it's original
    filename 

    Parameters
    ----------
    container : str
        unique filepath for the file to be saved
    url_template : 
        stem of URL 
    url_params_template : dict
        dict of parameter names and values to paste into URL

    Returns
    ----------
    downloaded : bool
        True if download successful, False otherwise.
    session : requests.session

    
Download a single file via HTTP get.
Build the url from parameters and save the file to dsik under it's original
filename

Parameters

container : str
unique filepath for the file to be saved
url_template :
stem of URL
url_params_template : dict
dict of parameter names and values to paste into URL

Returns

downloaded : bool
True if download successful, False otherwise.
Download a single file via HTTP get . Build the url from parameters and save the file to dsik under it ' s original filename Parameters container : str unique filepath for the file to be saved url_template : stem of URL url_params_template : dict dict of parameter names and values to paste into URL Returns downloaded : bool True if download successful False otherwise .
def download_request(
        source_name,
        start,
        end,
        session,
        filename,
        container,
        url_template,
        url_params_template,
        second=None):
    url_params = {}  
    if url_params_template:
        for key, value in url_params_template.items():
            url_params[key] = value.format(
                u_start=start,
                u_end=end,
                u_second=second
            )
        url = url_template
    else:
        url = url_template.format(
            u_start=start,
            u_end=end,
            u_second=second
        )
    resp = session.get(url, params=url_params)
    if source_name == 'PSE':
        if resp.text in ['Brak uprawnieñ', 'Brak uprawnień']:
            downloaded = False
            return downloaded, session
    try:
        original_filename = (
            resp.headers['content-disposition']
            .split('filename=')[-1]
            .replace('"', '')
            .replace(';', '')
        )
        logger.debug('Downloaded from URL: %s Original filename: %s',
                     resp.url, original_filename)
    except KeyError:
        if filename:
            original_filename = filename.format(u_start=start, u_end=end)
        else:
            logger.info(
                'original filename could neither be retrieved from server '
                'nor sources.yml'
            )
            original_filename = 'unknown_filename'
        logger.debug('Downloaded from URL: %s', resp.url)
    filepath = os.path.join(container, original_filename)
    with open(filepath, 'wb') as output_file:
        for chunk in resp.iter_content(1024):
            output_file.write(chunk)
    downloaded = True
    return downloaded, session
def download_request ( source_name , start , end , session , filename , container , url_template , url_params_template , second = None ) : url_params = { } if url_params_template : for key , value in url_params_template . items ( ) : url_params [ key ] = value . format ( u_start = start , u_end = end , u_second = second ) url = url_template else : url = url_template . format ( u_start = start , u_end = end , u_second = second ) resp = session . get ( url , params = url_params ) if source_name == 'PSE' : if resp . text in [ 'Brak uprawnieñ', Brak uprawnień']: downloaded = False return downloaded , session try : original_filename = ( resp . headers [ 'content-disposition' ] . split ( 'filename=' ) [ - 1 ] . replace ( '"' , '' ) . replace ( ';' , '' ) ) logger . debug ( 'Downloaded from URL: %s Original filename: %s' , resp . url , original_filename ) except KeyError : if filename : original_filename = filename . format ( u_start = start , u_end = end ) else : logger . info ( 'original filename could neither be retrieved from server ' 'nor sources.yml' ) original_filename = 'unknown_filename' logger . debug ( 'Downloaded from URL: %s' , resp . url ) filepath = os . path . join ( container , original_filename ) with open ( filepath , 'wb' ) as output_file : for chunk in resp . iter_content ( 1024 ) : output_file . write ( chunk ) downloaded = True return downloaded , session
Download a single file via HTTP get.
Download a single file via HTTP get .
"""
    Download a single file via HTTP get.
    Build the url from parameters and save the file to dsik under it's original
    filename 

    Parameters
    ----------
    container : str
        unique filepath for the file to be saved
    url_template : 
        stem of URL 
    url_params_template : dict
        dict of parameter names and values to paste into URL

    Returns
    ----------
    downloaded : bool
        True if download successful, False otherwise.
    session : requests.session

    """ # A dict for URL-parameters # For most sources, we can use HTTP get method with parameters-dict # For other sources that use urls without parameters (e.g. Svenska # Kraftnaet) # For polish TSO PSE, URLs have been guessed. # Don't proceed for wrong guesses # Get the original filename # For cases where the original filename can not be retrieved, # I put the filename in the param_dict # Save file to disk
source_name
start
end
session
filename
container
url_template
url_params_template
second