# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import re
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras.utils as ku
import warnings
warnings.filterwarnings('ignore')

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, Dense, LSTM, Bidirectional, Dropout
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import regularizers

# +
data = open('data.txt', encoding = 'utf = 8').read()

corpus = data.split('\n')
corpus
# -

len(corpus)

# +
cleaned_corpus = []
tokenizer = Tokenizer()

for line in corpus:
    res = re.sub('[!,*)@।#%(&$_?.^]', '', line)
    cleaned_corpus.append(res)
    
tokenizer.fit_on_texts(cleaned_corpus)
total_words = len(tokenizer.word_index) + 1 
# -

cleaned_corpus[:10]

# +
tokenizer = Tokenizer()

def get_sequence_of_tokens(corpus):
    ## Tokenization
    tokenizer.fit_on_texts(corpus)
    total_words = len(tokenizer.word_index) + 1 
    
    # convert data to sequence of tokens
    input_sequences = []
    for line in corpus:
        token_list = tokenizer.texts_to_sequences([line])[0]
        for i in range(1, len(token_list)):
            n_gram_sequence = token_list[:i+1]
            input_sequences.append(n_gram_sequence)
    return input_sequences, total_words

inp_sequences, total_words = get_sequence_of_tokens(cleaned_corpus)
# -

print(len(inp_sequences))
inp_sequences[:5]


# +
def generate_padded_sequences(input_sequences):
    max_sequence_len = max([len(x) for x in input_sequences])
    input_sequences = np.array(pad_sequences(input_sequences, maxlen = max_sequence_len, padding = 'pre'))
    
    predictors, label = input_sequences[:,:-1],input_sequences[:,-1]
    label = ku.to_categorical(label, num_classes = total_words)
    return predictors, label, max_sequence_len

predictors, label, max_sequence_len = generate_padded_sequences(inp_sequences)
# -

max_sequence_len


# +
def create_model(max_sequence_len, total_words):
    input_len = max_sequence_len - 1 
    
    model = Sequential()
    model.add(Embedding(total_words, 100, input_length = max_sequence_len))
    model.add(Bidirectional(LSTM(150, return_sequences = True)))
    model.add(Dropout(0.2))
    model.add(LSTM(100))
    
    model.add(Dense(total_words / 2, activation = 'relu', kernel_regularizer = regularizers.l2(0.01)))
    model.add(Dense(total_words, activation = 'softmax'))
    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
    return model

model = create_model(max_sequence_len, total_words)
model.summary()
# -

history = model.fit(predictors, label, epochs = 100, verbose = 1)

# ! pip install pydotplus

tf.keras.utils.plot_model(
    model,
    to_file="model.jpg",
    show_shapes=False,
    show_dtype=False,
    show_layer_names=True,
    rankdir="TB",
    expand_nested=False,
    dpi=100,
)

model.save('my_model.h5')

metrics = [k for k in history.history.keys() if ("loss" not in k) and ("val" not in k)]
fig, ax = plt.subplots(nrows=1, ncols=1, sharey=True , figsize=(5,5) , dpi = 200)
ax.set(title="Training")
ax11 = ax.twinx()
ax.plot(history.history['loss'], color='black')
ax.set_xlabel('Epochs')
ax.set_ylabel('Loss', color='black')
for metric in metrics:
    ax11.plot(history.history[metric], label=metric)
plt.show()


# ########################################### TESTING THE MODEL OUTPUT ###########################################

def generate_text(seed_text, next_words, model, max_sequence_len):
    for _ in range(next_words):
        token_list = tokenizer.texts_to_sequences([seed_text])[0]
        token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
        predicted = model.predict(token_list, verbose=0)
        predicted = np.argmax(predicted)
        output_word = ""
        for word,index in tokenizer.word_index.items():
            if index == predicted:
                output_word = word
                break
        seed_text += " "+output_word
    return seed_text.title()


print (generate_text("मनको", 5, model, max_sequence_len))
print (generate_text("आमा", 5, model, max_sequence_len))
print (generate_text("माया", 5, model, max_sequence_len))
print (generate_text("तिमी", 5, model, max_sequence_len))
# print (generate_text("new york", 4, model, max_sequence_len))
# # print (generate_text("science and technology", 5, model, max_sequence_len))

StarcoderdataJupyterScriptsDedupFiltered