import requests
import tarfile
import os

def download_file(url, directory):
    local_filename = os.path.join(directory, url.split('/')[-1])
    print ("Downloading %s --> %s"%(url, local_filename))
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
    return local_filename

def extract_tar(fpath):
    fname_dir, fname = os.path.split(fpath)
    dest_path = os.path.join(fname_dir,fname.split('.')[0])
    print ("Extracting %s --> %s"%(fpath, dest_path))
    if fname.endswith("tar.gz"):
        tar = tarfile.open(fpath, "r:gz")
        tar.extractall(path=fname_dir)
        tar.close()
    elif fname.endswith("tar"):
        tar = tarfile.open(fname, "r:")
        tar.extractall(path=fname_dir)
        tar.close()
    return dest_path

def list_files(startpath):
    for root, dirs, files in os.walk(startpath):
        level = root.replace(startpath, '').count(os.sep)
        indent = ' ' * 4 * (level)
        print('{}{}/'.format(indent, os.path.basename(root)))
        subindent = ' ' * 4 * (level + 1)
        for f in files:
            print('{}{}'.format(subindent, f))
[(16, 'os.path.split', 'os.path.split', False, 'import os\n'), (30, 'os.walk', 'os.walk', False, 'import os\n'), (8, 'requests.get', 'requests.get', False, 'import requests\n'), (20, 'tarfile.open', 'tarfile.open', False, 'import tarfile\n'), (24, 'tarfile.open', 'tarfile.open', False, 'import tarfile\n'), (33, 'os.path.basename', 'os.path.basename', False, 'import os\n')]