# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# %load_ext autoreload

# %autoreload 2

from pathlib import Path

from tqdm import tqdm

from news_utils.clean import german
# -

num_texts_per_file = 100000 # 100 000

input_dir = '/mnt/data/group07/johannes/germanlm/proc/final/'
output_dir = '/mnt/data/group07/johannes/germanlm/proc/final_merged/'


def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]


def merge_into_one(fns, i):
    with open(output_dir + str(i) + '.txt', 'w') as outfile:
        for fn in fns:
            text = Path(fn).read_text()
            text = german.clean_german(text)
            outfile.write(text + '\n')


Path(output_dir).mkdir(exist_ok=True)
for i, fns in tqdm(enumerate(chunks(list(Path(input_dir).glob('*.txt')), num_texts_per_file))):
    merge_into_one(fns, i)

StarcoderdataJupyterScriptsDedupFiltered