fdutils/web/web.py
def download_resources(*urls, local_folder='', chunk_size=1 << 15, session=None, replace=False, threads_count=10,
                       unzip=False, split_requests=False, timestamp=False, delete_on_bad_hash=True,
                       **request_kwargs):
    """ downloads one or more web/ftp resources, it also unzip them if they are archived in some known forms
        (gzip, bz2, zip, or tar.gz) and unzip=True, it will also check hash values of the downloaded files if specified.

        The downloads and unzipping are down in parallel using threads.
        For HTTP downloads it also supports HTTP Bytes range header to do partial downloads in case a failure occurs
        in the middle of a download

    Args:
        urls (*str or *HashURLResource): each should start with http(s):// or ftp://
        local_folder (str):
        chunk_size (int):
        session (requests.Session):
        replace (bool): whether to replace the file if same name or add index to it
        threads_count (int):
        unzip (bool): whether to unzip the file after received
        split_requests (bool): whether to use HTTP Byte Range header and do concurrent retrieval of its parts
        **request_kwargs: requests parameters if needed

    Returns:

    """
    url_resources = set(convert_url_to_hashedurlresource(url, assign_filename=(len(urls) == 1)) for url in urls)
    file_suffix_func = get_filename_indexed if not timestamp else get_filename_timestamped

    futures_res = {}
    ret = {}
    with futures.ThreadPoolExecutor(max_workers=threads_count) as executor:
        for url_resource in url_resources:

            if url_resource.is_ftp():
                log.debug('Download FTP: {}'.format(url_resource.url))
                submitted = executor.submit(ftp_download_hashed_resource, url_resource, local_folder, chunk_size,
                                            unzip, replace, file_suffix_func, delete_on_bad_hash)
            else:
                log.debug('Download HTTP(s): {}'.format(url_resource.url))
                submitted = executor.submit(http_download_hashed_resource, url_resource, local_folder, session,
                                            chunk_size, unzip, split_requests, replace, file_suffix_func,
                                            delete_on_bad_hash, **request_kwargs)

            futures_res[submitted] = url_resource

        for executed_future in futures.as_completed(futures_res):
            response = executed_future.result()
            if response:
                status_code, filename = response
                ret[futures_res[executed_future]] = dict(status_code=status_code, local_filename=filename)
            else:
                print('Problems with {}'.format(futures_res[executed_future]))

    return ret
 downloads one or more web/ftp resources, it also unzip them if they are archived in some known forms
        (gzip, bz2, zip, or tar.gz) and unzip=True, it will also check hash values of the downloaded files if specified.

        The downloads and unzipping are down in parallel using threads.
        For HTTP downloads it also supports HTTP Bytes range header to do partial downloads in case a failure occurs
        in the middle of a download

    Args:
        urls (*str or *HashURLResource): each should start with http(s):// or ftp://
        local_folder (str):
        chunk_size (int):
        session (requests.Session):
        replace (bool): whether to replace the file if same name or add index to it
        threads_count (int):
        unzip (bool): whether to unzip the file after received
        split_requests (bool): whether to use HTTP Byte Range header and do concurrent retrieval of its parts
        **request_kwargs: requests parameters if needed

    Returns:

    
downloads one or more web/ftp resources, it also unzip them if they are archived in some known forms
(gzip, bz2, zip, or tar.gz) and unzip=True, it will also check hash values of the downloaded files if specified.

The downloads and unzipping are down in parallel using threads.
For HTTP downloads it also supports HTTP Bytes range header to do partial downloads in case a failure occurs
in the middle of a download
downloads one or more web / ftp resources it also unzip them if they are archived in some known forms ( gzip bz2 zip or tar . gz ) and unzip = True it will also check hash values of the downloaded files if specified . The downloads and unzipping are down in parallel using threads . For HTTP downloads it also supports HTTP Bytes range header to do partial downloads in case a failure occurs in the middle of a download
def download_resources(*urls, local_folder='', chunk_size=1 << 15, session=None, replace=False, threads_count=10,
                       unzip=False, split_requests=False, timestamp=False, delete_on_bad_hash=True,
                       **request_kwargs):
    url_resources = set(convert_url_to_hashedurlresource(url, assign_filename=(len(urls) == 1)) for url in urls)
    file_suffix_func = get_filename_indexed if not timestamp else get_filename_timestamped
    futures_res = {}
    ret = {}
    with futures.ThreadPoolExecutor(max_workers=threads_count) as executor:
        for url_resource in url_resources:
            if url_resource.is_ftp():
                log.debug('Download FTP: {}'.format(url_resource.url))
                submitted = executor.submit(ftp_download_hashed_resource, url_resource, local_folder, chunk_size,
                                            unzip, replace, file_suffix_func, delete_on_bad_hash)
            else:
                log.debug('Download HTTP(s): {}'.format(url_resource.url))
                submitted = executor.submit(http_download_hashed_resource, url_resource, local_folder, session,
                                            chunk_size, unzip, split_requests, replace, file_suffix_func,
                                            delete_on_bad_hash, **request_kwargs)
            futures_res[submitted] = url_resource
        for executed_future in futures.as_completed(futures_res):
            response = executed_future.result()
            if response:
                status_code, filename = response
                ret[futures_res[executed_future]] = dict(status_code=status_code, local_filename=filename)
            else:
                print('Problems with {}'.format(futures_res[executed_future]))
    return ret
def download_resources ( * urls , local_folder = '' , chunk_size = 1 << 15 , session = None , replace = False , threads_count = 10 , unzip = False , split_requests = False , timestamp = False , delete_on_bad_hash = True , ** request_kwargs ) : url_resources = set ( convert_url_to_hashedurlresource ( url , assign_filename = ( len ( urls ) == 1 ) ) for url in urls ) file_suffix_func = get_filename_indexed if not timestamp else get_filename_timestamped futures_res = { } ret = { } with futures . ThreadPoolExecutor ( max_workers = threads_count ) as executor : for url_resource in url_resources : if url_resource . is_ftp ( ) : log . debug ( 'Download FTP: {}' . format ( url_resource . url ) ) submitted = executor . submit ( ftp_download_hashed_resource , url_resource , local_folder , chunk_size , unzip , replace , file_suffix_func , delete_on_bad_hash ) else : log . debug ( 'Download HTTP(s): {}' . format ( url_resource . url ) ) submitted = executor . submit ( http_download_hashed_resource , url_resource , local_folder , session , chunk_size , unzip , split_requests , replace , file_suffix_func , delete_on_bad_hash , ** request_kwargs ) futures_res [ submitted ] = url_resource for executed_future in futures . as_completed ( futures_res ) : response = executed_future . result ( ) if response : status_code , filename = response ret [ futures_res [ executed_future ] ] = dict ( status_code = status_code , local_filename = filename ) else : print ( 'Problems with {}' . format ( futures_res [ executed_future ] ) ) return ret
downloads one or more web/ftp resources, it also unzip them if they are archived in some known forms
(gzip, bz2, zip, or tar.gz) and unzip=True, it will also check hash values of the downloaded files if specified.
downloads one or more web / ftp resources it also unzip them if they are archived in some known forms ( gzip bz2 zip or tar . gz ) and unzip = True it will also check hash values of the downloaded files if specified .
""" downloads one or more web/ftp resources, it also unzip them if they are archived in some known forms
        (gzip, bz2, zip, or tar.gz) and unzip=True, it will also check hash values of the downloaded files if specified.

        The downloads and unzipping are down in parallel using threads.
        For HTTP downloads it also supports HTTP Bytes range header to do partial downloads in case a failure occurs
        in the middle of a download

    Args:
        urls (*str or *HashURLResource): each should start with http(s):// or ftp://
        local_folder (str):
        chunk_size (int):
        session (requests.Session):
        replace (bool): whether to replace the file if same name or add index to it
        threads_count (int):
        unzip (bool): whether to unzip the file after received
        split_requests (bool): whether to use HTTP Byte Range header and do concurrent retrieval of its parts
        **request_kwargs: requests parameters if needed

    Returns:

    """
None
local_folder
None
chunk_size
None
session
None
replace
whether to replace the file if same name or add index to it
whether to replace the file if same name or add index to it
threads_count
None
unzip
whether to unzip the file after received
whether to unzip the file after received
split_requests
whether to use HTTP Byte Range header and do concurrent retrieval of its parts
whether to use HTTP Byte Range header and do concurrent retrieval of its parts
timestamp
delete_on_bad_hash
urls
each should start with http(s):// or ftp.
each should start with http ( s ) : // or ftp .
**request_kwargs
requests parameters if needed
requests parameters if needed