# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] heading_collapsed=true
# # Deps

# + hidden=true
# %reload_ext autoreload
# %autoreload 2
# %matplotlib inline

# + hidden=true
from fastai import *
from fastai.text import *
from fastai.vision import *
from fastai.callbacks import *

# + hidden=true
txt_proc = [TokenizeProcessor(tokenizer=Tokenizer(lang='el') ),NumericalizeProcessor()]


# + hidden=true
def SampleTexts(df,max_tokens = 100000000):
    lens = np.array(df.apply(lambda x : len(re.findall(r'\b',x.text))/2,axis=1))
    n_tokens = np.sum(lens)
    if n_tokens <= max_tokens:
        return range(0,df.shape[0]), n_tokens
    #if np.sum(lens) > max_tokens:
    iDec = (-lens).argsort()
    cumSum = np.cumsum(lens[iDec])
    iCut = np.argmax(cumSum>max_tokens)
    return iDec[:iCut], cumSum[iCut-1]


# -

# # Greek Lanuage Model

# Extract the data from wikipedia using <a href="https://forums.fast.ai/t/language-model-zoo-gorilla/14623/19">this guide</a> by Moody.

path_root = Path('/home/jupyter/tutorials/')
source_txt = 'el_wiki_df_all.csv'
bs = 112

# + [markdown] heading_collapsed=true
# ## Preparing the data

# + [markdown] hidden=true
# The wiki text is contained in lots of seperate json files, so first we combine into a single large csv which we load with the data loader.
# We don't want to train with more than 100,000,000 tokens.  For the Greek corpus this is not a problem becuase there are much less than this number of tokens available.
# To sample we can reduce this number of tokens even futher is desired.

# + hidden=true
df = []
for file in path_lang_model.glob("el_dump/*/*"):
    df_file = pd.read_json(file,lines=True,encoding="utf-8")
    df.append(df_file)
df = pd.concat(df, axis=0)

# + hidden=true
df.to_csv(path_lang_model/wiki_103_csv,index=False)

# + hidden=true
df = pd.read_csv(path_lang_model/wiki_103_csv)

# + hidden=true
max_tokens = 100000000
i_keep,n_tokens = SampleTexts(df,max_tokens)

# + hidden=true
if len(i_keep) == df.shape[0]:
    print(f'{n_tokens} tokens < {max_tokens}, no need to sample')
else:
    df_sample = df.iloc[i_keep]
    df_sample.to_csv(path_lang_model/f'../el_wiki_df_{n_tokens}.csv',index=False)

# + [markdown] heading_collapsed=true
# ## Try small sample

# + [markdown] hidden=true
# ### Sample

# + hidden=true
df = pd.read_csv(path_root/'data'/wiki_103_csv)
max_tokens = 1000000

# + hidden=true
i_keep,n_tokens = SampleTexts(df,max_tokens)
df_sample = df.iloc[i_keep]
df_sample.to_csv(path_root/f'data/el_wiki_df_{n_tokens}.csv',index=False)

# + [markdown] hidden=true
# ### Tokenize for language model

# + [markdown] hidden=true
# Initially limit the vocab to 30k to compare with other ULMFit methods

# + hidden=true
source_txt = f'el_wiki_df_{n_tokens}.csv'
max_vocab = 30000

# + hidden=true
data_lm = (TextList.from_csv(path_root,f'data/{source_txt}',cols=1,
    processor = [TokenizeProcessor(tokenizer=Tokenizer(lang='el')) , NumericalizeProcessor(vocab=None, max_vocab=max_vocab)])
    .random_split_by_pct(0.1)
    .label_for_lm()
    .databunch(bs=bs))
print(f'Vocab size: {len(data_lm.vocab.itos)}')

# + hidden=true
data_lm.save('tmp_data_'+ source_txt)

# + [markdown] hidden=true
# ### Fit

# + hidden=true
data_lm = TextLMDataBunch.load(path_root, 'tmp_data_'+ source_txt ,bs=bs)

# + hidden=true
learn = language_model_learner(data_lm, drop_mult=0.3)

# + hidden=true
learn.lr_find()

# + hidden=true
learn.recorder.plot(skip_end=10)

# + hidden=true
learn.fit_one_cycle(10, 1e-2, moms=(0.8,0.7),callbacks=[ShowGraph(learn), SaveModelCallback(learn,monitor='accuracy',mode='max')])
# -

# ## Process full text

# + [markdown] heading_collapsed=true
# ### Tokenize for language model

# + [markdown] hidden=true
# Initially limit the vocab to 30k to compare with other ULMFit methods

# + hidden=true
max_vocab = 30000

# + hidden=true
data_lm = (TextList.from_csv(path_root,f'data/{source_txt}',cols=1,
    processor = [TokenizeProcessor(), NumericalizeProcessor(vocab=None, max_vocab=max_vocab)])
    .random_split_by_pct(0.1)
    .label_for_lm()
    .databunch(bs=bs))
print(f'Vocab size: {len(data_lm.vocab.itos)}')
data_lm.save('tmp_data_lm'+ source_txt)

# + hidden=true
#save the dictionary
pickle_out = open(str(path_root/'data/dict_')+ source_txt + '.pkl',"wb")
pickle.dump(data_lm.vocab.itos, pickle_out)
pickle_out.close()
# -

# ### Fit

data_lm = TextLMDataBunch.load(path_root, 'tmp_data_lm'+ source_txt ,bs=bs)

learn = language_model_learner(data_lm, drop_mult=0.1)

# we were pre-empted by goole, lets keep going for a bit
learn.load('bestmodel_lm4_' + source_txt);

learn.lr_find()

learn.recorder.plot(skip_end=20)

learn.fit_one_cycle(30, 1e-4, moms=(0.8,0.7),callbacks=[ShowGraph(learn), SaveModelCallback(learn,monitor='accuracy',mode='max',name='bestmodel_lm5_' + source_txt)])

print(f'Perplexity for best validation accuracy: {math.exp(2.678463)}')

learn.fit_one_cycle(30, 5e-3, moms=(0.8,0.7),callbacks=[ShowGraph(learn), SaveModelCallback(learn,monitor='accuracy',mode='max',name='bestmodel_lm4_' + source_txt)])

StarcoderdataJupyterScriptsDedupFiltered