# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from fastai.text import *
import numpy as np
from sklearn.model_selection import train_test_split
import pickle
import sentencepiece as spm
import re
import pdb

import fastai, torch
fastai.__version__ , torch.__version__

torch.cuda.set_device(0)

path = Path('/home/gaurav/Downloads/ClassificationModel')

df = pd.read_csv(path/'Punjabi_Binary_Classification.csv')
df.head()

print('Positive Examples', df['ਭਾਰਤੀ ਰਾਜਨੀਤੀ'].sum())
print('Negative Examples', df.shape[0] -df['ਭਾਰਤੀ ਰਾਜਨੀਤੀ'].sum())

df['text'][0]

df.shape

df[df['text'].isnull()].shape

label_cols = list(range(1,df.shape[1]))

print(label_cols)

id_to_col = list(df.columns)

col_to_id = {}
for i, col in enumerate(id_to_col):
    col_to_id[col] = i


class PanjabiTokenizer(BaseTokenizer):
    def __init__(self, lang:str):
        self.lang = lang
        self.sp = spm.SentencePieceProcessor()
        self.sp.Load(str('/home/gaurav/Downloads/LanguageModel/panjabi_lm.model'))
        
    def tokenizer(self, t:str) -> List[str]:
        return self.sp.EncodeAsPieces(t)


sp = spm.SentencePieceProcessor()
sp.Load('/home/gaurav/Downloads/LanguageModel/panjabi_lm.model')
itos = [sp.IdToPiece(int(i)) for i in range(30000)]

itos

# 30,000 is the vocab size that we chose in sentencepiece
panjabi_vocab = Vocab(itos)

panjabi_tok = PanjabiTokenizer('pb')

tokenizer = Tokenizer(tok_func=PanjabiTokenizer, lang='pb')

tokenizer.special_cases

data_lm = TextDataBunch.from_csv(path=path, csv_name='Punjabi_Binary_Classification.csv', text_cols=[0], label_cols=label_cols, tokenizer=tokenizer, vocab=panjabi_vocab)

data_lm.save()

bs=48

data_lm = TextLMDataBunch.load(path, 'tmp', bs=bs)

data_lm.show_batch()

learn = language_model_learner(data_lm, drop_mult=0.3)

# Loading the pretrained language model on panjabi wikipedia
learn.load('/home/gaurav/Downloads/LanguageModel/PanjabiDataset/models/fourth', with_opt=True)

# +
# Fine tuning the prtrained LM on current dataset
# -

learn.lr_find()

learn.recorder.plot()

learn.freeze()

learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))

learn.save('fit_head', with_opt=True)

learn.load('fit_head', with_opt=True);

learn.unfreeze()

learn.fit_one_cycle(5, 1e-3, moms=(0.8,0.7))

learn.save('fine_tuned', with_opt=True)

learn.load('fine_tuned', with_opt=True);

learn.predict('ਜੋ ਉਹਨਾਂ ਦੇ ਰੱਬਾਂ ਨੂੰ ਪ੍ਰਸਤੁਤ ਕਰਦੇ',n_words=10)

learn.save_encoder('fine_tuned_enc')

data_clas = TextClasDataBunch.from_csv(path=path, csv_name='Punjabi_Binary_Classification.csv', tokenizer=tokenizer, vocab=panjabi_vocab, text_cols=[0], label_cols=label_cols)

data_clas.show_batch()

data_clas.sanity_check()

learn = text_classifier_learner(data_clas, drop_mult=0.5)

learn.load_encoder('fine_tuned_enc')

learn.freeze()

learn.lr_find()

learn.recorder.plot()

learn.loss_func.func

kappa = KappaScore()

learn.metrics = [kappa]

learn.fit_one_cycle(1, 1e-1, moms=(0.8,0.7))

learn.save('first-full')

learn.load('first-full');

learn.freeze_to(-2)
learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))

learn.unfreeze()
learn.fit_one_cycle(5, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))

# +
# Todos: Clean the input text a little

StarcoderdataJupyterScriptsDedupFiltered