# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernel_info:
#     name: ml-software
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import internetarchive as ia
import pandas as pd
import itertools as it
from toolz import pluck, filter, map, take
import toolz
import os
from pathlib import Path
import json
from glob import glob
from zipfile import ZipFile, BadZipFile

config = json.load(open(os.path.expanduser("~/.thesis.conf")))
db_folder = Path(config['datasets']) / Path("archive_org/") 
os.chdir(str(db_folder))

iaformat = ["Single Page Processed JP2 ZIP", 'Metadata']

search = ia.search_items('pages:[20 TO 25] AND (language:eng OR language:"English") AND date:[1800-01-01 TO 1967-01-01]')

items = list(toolz.take(5,search.iter_as_items()))

item = items[2]

for item in items[0:3]:
    ia.download(item.identifier, formats=iaformat)

jp2path = Path(item.identifier) / Path(next(pluck('name',filter(lambda files: files['format'] == iaformat, item.files))))

# + inputHidden=false outputHidden=false
zips = glob(str(db_folder / '*' / '*_jp2.zip'))
zips
# -

for zip in zips:
    try:
        jp2zip = ZipFile(str(zip))
        print(jp2zip.filename, len(jp2zip.namelist()))
        jp2zip.extract(jp2zip.namelist()[0])
    except BadZipFile:
        print('zip error', zip)

jp2zip.namelist()





StarcoderdataJupyterScriptsDedupFiltered