import requests
import tarfile
import os

def download_file(url, directory):
    local_filename = os.path.join(directory, url.split('/')[-1])
    print ("Downloading %s --> %s"%(url, local_filename))
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
    return local_filename

def extract_tar(fpath):
    fname_dir, fname = os.path.split(fpath)
    dest_path = os.path.join(fname_dir,fname.split('.')[0])
    print ("Extracting %s --> %s"%(fpath, dest_path))
    if fname.endswith("tar.gz"):
        tar = tarfile.open(fpath, "r:gz")
        tar.extractall(path=fname_dir)
        tar.close()
    elif fname.endswith("tar"):
        tar = tarfile.open(fname, "r:")
        tar.extractall(path=fname_dir)
        tar.close()
    return dest_path

def list_files(startpath):
    for root, dirs, files in os.walk(startpath):
        level = root.replace(startpath, '').count(os.sep)
        indent = ' ' * 4 * (level)
        print('{}{}/'.format(indent, os.path.basename(root)))
        subindent = ' ' * 4 * (level + 1)
        for f in files:
            print('{}{}'.format(subindent, f))
[((16, 23, 16, 43), 'os.path.split', 'os.path.split', ({(16, 37, 16, 42): 'fpath'}, {}), '(fpath)', False, 'import os\n'), ((30, 29, 30, 47), 'os.walk', 'os.walk', ({(30, 37, 30, 46): 'startpath'}, {}), '(startpath)', False, 'import os\n'), ((8, 9, 8, 39), 'requests.get', 'requests.get', (), '', False, 'import requests\n'), ((20, 14, 20, 41), 'tarfile.open', 'tarfile.open', ({(20, 27, 20, 32): 'fpath', (20, 34, 20, 40): '"""r:gz"""'}, {}), "(fpath, 'r:gz')", False, 'import tarfile\n'), ((24, 14, 24, 39), 'tarfile.open', 'tarfile.open', ({(24, 27, 24, 32): 'fname', (24, 34, 24, 38): '"""r:"""'}, {}), "(fname, 'r:')", False, 'import tarfile\n'), ((33, 37, 33, 59), 'os.path.basename', 'os.path.basename', ({(33, 54, 33, 58): 'root'}, {}), '(root)', False, 'import os\n')]