archiver/util.py
def download_url(url):
    """download a given url's content into downloads/domain.txt"""

    if not os.path.exists(SOURCES_DIR):
        os.makedirs(SOURCES_DIR)

    ts = str(datetime.now().timestamp()).split('.', 1)[0]

    source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(domain(url), ts))

    print('[*] [{}] Downloading {} > {}'.format(
        datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        url,
        pretty_path(source_path),
    ))
    end = progress(TIMEOUT, prefix='      ')
    try:
        downloaded_xml = requests.get(url).content.decode()
        end()
    except Exception as e:
        end()
        print('[!] Failed to download {}\n'.format(url))
        print('    ', e)
        raise SystemExit(1)

    with open(source_path, 'w', encoding='utf-8') as f:
        f.write(downloaded_xml)

    return source_path
download a given url's content into downloads/domain.txt
download a given url's content into downloads/domain.txt
download a given url ' s content into downloads / domain . txt
def download_url(url):
    if not os.path.exists(SOURCES_DIR):
        os.makedirs(SOURCES_DIR)
    ts = str(datetime.now().timestamp()).split('.', 1)[0]
    source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(domain(url), ts))
    print('[*] [{}] Downloading {} > {}'.format(
        datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        url,
        pretty_path(source_path),
    ))
    end = progress(TIMEOUT, prefix='      ')
    try:
        downloaded_xml = requests.get(url).content.decode()
        end()
    except Exception as e:
        end()
        print('[!] Failed to download {}\n'.format(url))
        print('    ', e)
        raise SystemExit(1)
    with open(source_path, 'w', encoding='utf-8') as f:
        f.write(downloaded_xml)
    return source_path
def download_url ( url ) : if not os . path . exists ( SOURCES_DIR ) : os . makedirs ( SOURCES_DIR ) ts = str ( datetime . now ( ) . timestamp ( ) ) . split ( '.' , 1 ) [ 0 ] source_path = os . path . join ( SOURCES_DIR , '{}-{}.txt' . format ( domain ( url ) , ts ) ) print ( '[*] [{}] Downloading {} > {}' . format ( datetime . now ( ) . strftime ( '%Y-%m-%d %H:%M:%S' ) , url , pretty_path ( source_path ) , ) ) end = progress ( TIMEOUT , prefix = '      ' ) try : downloaded_xml = requests . get ( url ) . content . decode ( ) end ( ) except Exception as e : end ( ) print ( '[!] Failed to download {}\n' . format ( url ) ) print ( '    ' , e ) raise SystemExit ( 1 ) with open ( source_path , 'w' , encoding = 'utf-8' ) as f : f . write ( downloaded_xml ) return source_path
download a given url's content into downloads/domain.txt
download a given url ' s content into downloads / domain . txt
"""download a given url's content into downloads/domain.txt"""
url