# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# fopen = open('dumping-common-crawl.txt')
# results = [next(fopen) for _ in range(1000000)]
# results[:1000]

# +
# # !wget https://f000.backblazeb2.com/file/malay-dataset/dumping/common-crawl/mse-index.zip
# # !unzip mse-index.zip
# -

import pandas as pd

# +
# df = pd.read_feather('data/cdx-00017.feather')
# -

from glob import glob
files = glob('feather/*.feather')
len(files)

# +
# # !wget https://f000.backblazeb2.com/file/malay-dataset/dumping/common-crawl/feather.zip
# # !unzip feather

# +
import dask
import feather
import dask.dataframe as dd
import dask.bag as db
import itertools

dask.__version__


# +
# from dask.distributed import Client

# client = Client(n_workers=16)
# client

# dfs = [dask.delayed(pd.read_feather)(f) for f in files]
# df = dd.from_delayed(dfs)

# +
# %%time

def get_tld(file):
    df = pd.read_feather(file)
    return df.groupby('tld')['sentence'].apply(list).reset_index()

bags = db.from_sequence(files)
mapped = bags.map(get_tld)
computed = mapped.compute(num_workers = 20)
# -

computed[0]

# +
import itertools

def nested_list(rows):
    merged = list(itertools.chain(*rows))
    merged = list(filter(None, merged))
    merged = list(set(merged))
    return merged

concated = pd.concat(computed)
agg = concated.groupby('tld').agg({'sentence': nested_list})
# -

for i in range(len(agg)):
    print(i, agg.index[i], agg['sentence'].iloc[i][:10], '\n')

archi, asia, at, audio, ax, band, bc.ca, bg, bid, biz, biz.id, blog, business, bz
ca, cafe, cat, cc, center, cf, ch, church, city, cl, click, cloud, cm, cn,
co, co.id, co.in, co.jp, co.kr, co.nz, com, com.cau, com.bn, com.br, com.cn,
com.co, com.cy, com.mx, com.my, com.pl, com.py, coop, cz, desa.id, digital,
direct, edu, edu.cn, edu.my, ee, es, eu, fi, fm, fr, fun, ga, global, go.id, go.jp,
gov, gov.bn, gov.mo, gov.my, gq, gr, graphics, gratis, help, hk, hu, im, international,
io, ir, is, it, jp, kg, la, lg.jp, li, link, live, lt, lv, ma, men, mil.id, mil.my,


StarcoderdataJupyterScriptsDedupFiltered