# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %load_ext autoreload
# %autoreload 2

import json
import os

import scraper_bremen

USE_CACHE = True

if USE_CACHE:
    os.makedirs('./_cache', exist_ok=True)

with open('../bundesrat/sessions.json') as f:
    sessions = json.load(f)
len(sessions)

FILENAME = 'session_tops.json'
if os.path.exists(FILENAME):
    with open(FILENAME) as f:
        session_tops = json.load(f)
else:
    session_tops = {}

# +
for session in sessions:
    num = session['number']
    if str(num) in session_tops:
        continue
    print('\nLoading tops of: %s' % num)
    
    result = scraper_bremen.MainExtractorMethod(scraper_bremen.TextExtractorHolder).get_session(session)
    if result is None:
        continue
    session_tops[str(num)] = result
    with open(FILENAME, 'w') as f:
        json.dump(session_tops, f)
        
print('Total sessions:', len(session_tops))
# -



StarcoderdataJupyterScriptsDedupFiltered