# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.7.3 64-bit
#     language: python
#     name: python37364bit3ca3215d4e6240698248485fce128d4d
# ---

# +
# External libraries
import requests
import nbimporter

# Builtins
import json
import os
import concurrent.futures
import functools
import zipfile
import pathlib
import urllib.request
# -

# ### Establish sprites url

# +
db_path = "./data/pokemon_details_pokeapi/"

# Make directries (recursively) if not exist.
pathlib.Path(db_path).mkdir(parents=True, exist_ok=True)

sprite_urls = []
for file in os.listdir(db_path):
    with open(db_path + file) as f:
        data = json.load(f)
        if data["sprites"]["front_default"]:
            sprite_urls.append((data["id"], data["species"]["name"] , data["sprites"]["front_default"]))
print(f"There are {len(sprite_urls)} sprites in total")


# -

# `sprite_urls` looks like this
# ```python
# [(10001, 'deoxys', 'https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/10001.png'),
#  (10002, 'deoxys', 'https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/10002.png'),
#  (10003, 'deoxys', 'https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/10003.png'),
#  ...
# ]
# ```

# ### Download Sprites using 10 threads

def download_sprite(sprite_url, overwrite=False):
    id, name, url = sprite_url
    path = f"./data/pokemon_sprites_pokeapi/{id}_{name}.png"
    if overwrite or not os.path.exists(path):
        urllib.request.urlretrieve(url, path)


# Multithreading
no_threads = 10
with concurrent.futures.ThreadPoolExecutor(max_workers=no_threads) as executor:
    executor.map(download_sprite, sprite_urls)
print(f"Finish downloading all sprites")

# ### Make a zip file for archiving

path = "./data/pokemon_sprites_pokeapi/"
with zipfile.ZipFile(f'{path[:-1]}.zip','w') as zip_file: 
    for file in os.listdir(f"{path}"):
        zip_file.write(f"{path}{file}", f"{file}", compress_type=zipfile.ZIP_DEFLATED)
print(f"Zip to {path[:-1]}.zip successfully")



StarcoderdataJupyterScriptsDedupFiltered