from multiprocessing import Pool, cpu_count
from functools import partial
import urllib.request
from time import time

count = 0
def download_image(url, base=True):
    global count
    try:
        img_path = os.path.join("/kaggle/tmp/images", os.path.basename(url).strip("\n"))
        urllib.request.urlretrieve(url, img_path)
        print(f"Downloaded {count}: {img_path}")
    except Exception as e:
        print("[ERROR]: ", e)

if __name__ == "__main__":
    with open("/kaggle/tmp/urls.txt", "r") as f:
        urls = f.readlines()

    print(f"{cpu_count()} CPUs available")
    pool = Pool(cpu_count())
    download_func = partial(download_image, base=True)
    start = time()
    results = pool.map(download_func, urls)
    pool.close()
    pool.join()
    end = time()

    print(f"[INFO] Took {end-start}s")