apache_downloader/downloader.py
def download_and_verify(path, destination=None):
    """
    Downloads the Apache file and verifies its hash
    :param path: the download file path, e.g. /nifi/nifi-registry/nifi-registry-0.5.0/nifi-registry-0.5.0-bin.tar.gz
    :param destination: the location to save the downloaded file or file object
    """
    destination = destination or "."
    if isinstance(destination, str):
        download_dir = dirname(destination)
        download_file = basename(destination) or basename(path)
        if isdir(download_file):
            download_dir = download_file
            download_file = basename(path)
        download_path = expanduser(os.path.join(download_dir, download_file))
        logging.info("Downloading Apache project {path} to destination {dest}".format(path=path, dest=download_path))
        assert not os.path.exists(download_path), "File already exists"
    else:
        download_path = destination
        logging.info("Downloading Apache project {path}".format(path=path))

    site = "www"
    try:
        expected_hash = get_hash(path, site)
    except requests.exceptions.HTTPError:
        logging.debug("Not found, try from archive")
        site = "archive"
        expected_hash = get_hash(path, site)
        logging.info("Downloading from archive")

    with requests.get(get_mirror_url(path, site), stream=True) as r:
        r.raise_for_status()
        file_length = r.headers.get("content-length")
        if file_length:
            file_length = int(file_length)
            logging.info("File size: {size}".format(size=humanize.naturalsize(file_length)))
            progress_bar = FillingCirclesBar("Downloading", max=ceil(file_length / DOWNLOAD_CHUNK_SIZE))
        else:
            progress_bar = Spinner("Downloading")

        def save_to_file(_f):
            m = hashlib.sha512()
            for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
                if chunk:
                    _f.write(chunk)
                    m.update(chunk)
                    progress_bar.next()
            actual_hash = m.hexdigest()
            assert actual_hash in expected_hash,\
                "Hash of downloaded file is invalid, expected {expected_hash} but got {actual_hash}.".format(
                    expected_hash=expected_hash,
                    actual_hash=actual_hash
                )

        if hasattr(download_path, "write"):
            save_to_file(download_path)
        else:
            with open(download_path, "wb") as f:
                save_to_file(f)
            assert os.path.exists(download_path), "File could not be downloaded."
    print(" Done.", file=sys.stderr)

    Downloads the Apache file and verifies its hash
    :param path: the download file path, e.g. /nifi/nifi-registry/nifi-registry-0.5.0/nifi-registry-0.5.0-bin.tar.gz
    :param destination: the location to save the downloaded file or file object
    
Downloads the Apache file and verifies its hash
Downloads the Apache file and verifies its hash
def download_and_verify(path, destination=None):
    destination = destination or "."
    if isinstance(destination, str):
        download_dir = dirname(destination)
        download_file = basename(destination) or basename(path)
        if isdir(download_file):
            download_dir = download_file
            download_file = basename(path)
        download_path = expanduser(os.path.join(download_dir, download_file))
        logging.info("Downloading Apache project {path} to destination {dest}".format(path=path, dest=download_path))
        assert not os.path.exists(download_path), "File already exists"
    else:
        download_path = destination
        logging.info("Downloading Apache project {path}".format(path=path))
    site = "www"
    try:
        expected_hash = get_hash(path, site)
    except requests.exceptions.HTTPError:
        logging.debug("Not found, try from archive")
        site = "archive"
        expected_hash = get_hash(path, site)
        logging.info("Downloading from archive")
    with requests.get(get_mirror_url(path, site), stream=True) as r:
        r.raise_for_status()
        file_length = r.headers.get("content-length")
        if file_length:
            file_length = int(file_length)
            logging.info("File size: {size}".format(size=humanize.naturalsize(file_length)))
            progress_bar = FillingCirclesBar("Downloading", max=ceil(file_length / DOWNLOAD_CHUNK_SIZE))
        else:
            progress_bar = Spinner("Downloading")
        def save_to_file(_f):
            m = hashlib.sha512()
            for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
                if chunk:
                    _f.write(chunk)
                    m.update(chunk)
                    progress_bar.next()
            actual_hash = m.hexdigest()
            assert actual_hash in expected_hash,\
                "Hash of downloaded file is invalid, expected {expected_hash} but got {actual_hash}.".format(
                    expected_hash=expected_hash,
                    actual_hash=actual_hash
                )
        if hasattr(download_path, "write"):
            save_to_file(download_path)
        else:
            with open(download_path, "wb") as f:
                save_to_file(f)
            assert os.path.exists(download_path), "File could not be downloaded."
    print(" Done.", file=sys.stderr)
def download_and_verify ( path , destination = None ) : destination = destination or "." if isinstance ( destination , str ) : download_dir = dirname ( destination ) download_file = basename ( destination ) or basename ( path ) if isdir ( download_file ) : download_dir = download_file download_file = basename ( path ) download_path = expanduser ( os . path . join ( download_dir , download_file ) ) logging . info ( "Downloading Apache project {path} to destination {dest}" . format ( path = path , dest = download_path ) ) assert not os . path . exists ( download_path ) , "File already exists" else : download_path = destination logging . info ( "Downloading Apache project {path}" . format ( path = path ) ) site = "www" try : expected_hash = get_hash ( path , site ) except requests . exceptions . HTTPError : logging . debug ( "Not found, try from archive" ) site = "archive" expected_hash = get_hash ( path , site ) logging . info ( "Downloading from archive" ) with requests . get ( get_mirror_url ( path , site ) , stream = True ) as r : r . raise_for_status ( ) file_length = r . headers . get ( "content-length" ) if file_length : file_length = int ( file_length ) logging . info ( "File size: {size}" . format ( size = humanize . naturalsize ( file_length ) ) ) progress_bar = FillingCirclesBar ( "Downloading" , max = ceil ( file_length / DOWNLOAD_CHUNK_SIZE ) ) else : progress_bar = Spinner ( "Downloading" ) def save_to_file ( _f ) : m = hashlib . sha512 ( ) for chunk in r . iter_content ( chunk_size = DOWNLOAD_CHUNK_SIZE ) : if chunk : _f . write ( chunk ) m . update ( chunk ) progress_bar . next ( ) actual_hash = m . hexdigest ( ) assert actual_hash in expected_hash , "Hash of downloaded file is invalid, expected {expected_hash} but got {actual_hash}." . format ( expected_hash = expected_hash , actual_hash = actual_hash ) if hasattr ( download_path , "write" ) : save_to_file ( download_path ) else : with open ( download_path , "wb" ) as f : save_to_file ( f ) assert os . path . exists ( download_path ) , "File could not be downloaded." print ( " Done." , file = sys . stderr )
Downloads the Apache file and verifies its hash
Downloads the Apache file and verifies its hash
"""
    Downloads the Apache file and verifies its hash
    :param path: the download file path, e.g. /nifi/nifi-registry/nifi-registry-0.5.0/nifi-registry-0.5.0-bin.tar.gz
    :param destination: the location to save the downloaded file or file object
    """
path
the download file path, e.g. /nifi/nifi-registry/nifi-registry-0.5.0/nifi-registry-0.5.0-bin.tar.gz
the download file path e . g . / nifi / nifi - registry / nifi - registry - 0 . 5 . 0 / nifi - registry - 0 . 5 . 0 - bin . tar . gz
destination
the location to save the downloaded file or file object
the location to save the downloaded file or file object