Tiny image scraper for xkcd.com This came out on the spur of the moment, as a quick and dirty tool to have the job done.

This is a simple image scraper for immensely popular and legendary comic website inspired by a Python Easter Egg.

For those who don't know it, run your Python interpreter and type import antigravity and hit Enter. :)

As for the code below, I'd appreciate any feedback, particularly in regards to threading, as I'm new to this.

#!/usr/bin/python3

import os
import sys
import time
import threading
from pathlib import Path
from shutil import copyfileobj


import requests
from lxml import html


BASE_URL = "https://www.xkcd.com/"
ARCHIVE = "https://www.xkcd.com/archive"
SAVE_DIRECTORY = Path('xkcd_comics')
LOGO = """
       _           _                      
 tiny | |  image  | | downloader for
 __  _| | _____ __| |  ___ ___  _ __ ___  
 \ \/ / |/ / __/ _` | / __/ _ \| '_ ` _ \ 
  &gt;  &lt;|   &lt; (_| (_| || (_| (_) | | | | | |
 /_/\_\_|\_\___\__,_(_)___\___/|_| |_| |_|
 version 0.1
"""


def show_logo():
    print(LOGO)


def fetch_url(url: str) -&gt; requests.Response:
    return requests.get(url)


def head_option(values: list) -&gt; str:
    return next(iter(values), None)


def get_penultimate(url: str) -&gt; int:
    page = fetch_url(url)
    tree = html.fromstring(page.content)
    newest_comic = head_option(
        tree.xpath('//*[@id="middleContainer"]/a[1]/@href'))
    return int(newest_comic.replace("/", ""))


def get_images_from_page(url: str) -&gt; str:
    page = fetch_url(url)
    tree = html.fromstring(page.content)
    return head_option(tree.xpath('//*[@id="comic"]//img/@src'))


def get_number_of_pages(latest_comic: int) -&gt; int:
    print(f"There are {latest_comic} comics.")
    print(f"How many do you want to download? Type 0 to exit.")
    while True:
        try:
            number_of_comics = int(input("&gt;&gt; "))
        except ValueError:
            print("Error: Expected a number. Try again.")
            continue
        if number_of_comics &gt; latest_comic or number_of_comics &lt; 0:
            print("Error: Incorrect number of comics. Try again.")
            continue
        elif number_of_comics == 0:
            sys.exit()
        return number_of_comics


def clip_url(img: str) -&gt; str:
    return img.rpartition("/")[-1]


def make_dir():
    return os.makedirs(SAVE_DIRECTORY, exist_ok=True)


def save_image(img: str):
    comic_name = clip_url(img)
    print(f"Downloading: {comic_name}")
    f_name = SAVE_DIRECTORY / comic_name
    with requests.get("https:" + img, stream=True) as img, open(f_name, "wb") \
            as output:
        copyfileobj(img.raw, output)


def show_time(seconds: int) -&gt; int:
    minutes, seconds = divmod(seconds, 60)
    hours, minutes = divmod(minutes, 60)
    time_elapsed = f"{hours:02d}:{minutes:02d}:{seconds:02d}" 
    return time_elapsed


def get_xkcd():
    show_logo()
    make_dir()

    collect_garbage = []
    latest_comic = get_penultimate(ARCHIVE)
    pages = get_number_of_pages(latest_comic)

    start = time.time()
    for page in reversed(range(latest_comic - pages + 1, latest_comic + 1)):
        print(f"Fetching page {page} out of {latest_comic}")
        try:
            url = get_images_from_page(f"{BASE_URL}{page}/")
            thread = threading.Thread(target=save_image, args=(url, ))
            thread.start()
        except (ValueError, AttributeError, requests.exceptions.MissingSchema):
            print(f"WARNING: Invalid comic image source url.")
            collect_garbage.append(f"{BASE_URL}{page}")
            continue
    thread.join()
    end = time.time()

    print(f"Downloaded {pages} comic(s) in {show_time(int(end - start))}.")

    if len(collect_garbage) &gt; 0:
        print("However, was unable to download images for these pages:")
        print("\n".join(page for page in collect_garbage))


def main():
    get_xkcd()


if __name__ == '__main__':
    main()
Stop hammering XKCD server


You basically wrote a denial-of-service attack, since you try to connect as fast and as often as possible to the server.
By sharing this code on CodeReview, it becomes a distributed denial-of-service.


Just in case it wasn't clear : don't do that.

I suppose the load of XKCD servers is high enough that your code won't make any noticeable difference. Still, the primary goal of your code should be to stay under the radar. Seen from outside, there shouldn't be any difference between launching your script and casually browsing https://xkcd.com/.

It isn't just as a courtesy to fellow developers : you might be banned by the remote servers if you send too many requests too fast.

Solutions


Wait at least one second before downloading any picture. The desired pause might be defined as Crawl-delay in robots.txt.
Remove multi-threading or at least limit the number of threads.
Don't download a file again if it is already in SAVE_DIRECTORY. You can check if it's here, if its size is larger than 0 or if it is indeed a PNG file.


Unrelated note

You know the id for each comic. You should probably write it in the filename : geologic_time.png could be called 2187_geologic_time.png or 02187_geologic_time.png.

This way, comics are sorted chronologically inside SAVE_DIRECTORY.