# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Initial setup

# %reload_ext autoreload
# %autoreload 2
# %matplotlib inline

from fastai import *
from fastai.text import *

# +
# import fastai.utils.collect_env
# fastai.utils.collect_env.show_install()
# -

bs = 256

# ## Prepare data

# +
data_path = Config.data_path()
lang = 'nl'
name = f'{lang}wiki'
path = data_path/name
mdl_path = path/'models'
lm_fns = [f'{lang}_wt', f'{lang}_wt_vocab']

# The language model was previously saved like this:
# learn.save(mdl_path/lm_fns[0], with_opt=False)
# learn.data.vocab.save(mdl_path/(lm_fns[1] + '.pkl'))
# -

sa_path = path/'110kDBRD'

# Takes ~ 6 minutes:
data_lm = (TextList.from_folder(sa_path)
          .filter_by_folder(include=['train', 'test', 'unsup'])
          .split_by_rand_pct(0.1, seed=42)
          .label_for_lm()
          .databunch(bs=bs, num_workers=1))
len(data_lm.vocab.itos), len(data_lm.train_ds)


data_lm.save('lm_databunch')

data_lm = load_data(sa_path, 'lm_databunch', bs=bs)

# learn_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=1., 
#                                   path = path, 
#                                   pretrained_fnames= ['lm_best', 'itos']).to_fp16()
learn_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=1., 
                                  path = path, 
                                  pretrained_fnames= ['lm_best', 'itos']).to_fp16()

# +
# language_model_learner??

# +
# Learner??
# -



StarcoderdataJupyterScriptsDedupFiltered