from google.colab import drive
drive.mount('/content/drive')
## Can be used for removing التشكيل 
!pip install pyarabic
import tensorflow as tf
import numpy as np
import pandas as pd
import os
from tqdm import tqdm
import time
import glob
from random import shuffle
from pyarabic import araby
from tensorflow.keras.layers import GRU, Embedding, Dense, Input, Dropout, Bidirectional, BatchNormalization, Flatten, Reshape
from tensorflow.keras.models import Sequential
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from sklearn.utils import shuffle
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
!pip install --upgrade --force-reinstall --no-deps kaggle
!mkdir -p ~/.kaggle
!cp kaggle.json ~/.kaggle/
!chmod 600 ~/.kaggle/kaggle.json
!kaggle competitions download -c machathon2-qualifications
import zipfile
filename = "machathon2-qualifications.zip"
with zipfile.ZipFile(filename, 'r') as zipf:
  zipf.extractall()
  print("Done")
with open('labels.txt', 'r') as f:
  label2name = f.readlines()
  label2name = [name.replace('\n', '') for name in label2name]
import pandas as pd
## It was enough to test the data with 100 examples as the data was convenient
data = pd.read_csv('train.csv')
train_data = data[:19900]  
test_data = data[19900:]
x_train = np.array(train_data['data'])
y_train  = np.array(train_data['labels'])

x_test = np.array(test_data['data'])
y_test = np.array(test_data['labels'])
## To exclude any useless parts in the "شطر" or "البيت"
execluded = '!()*-ـ.:=o[]«»;؛,،~?؟\u200f\ufeffـ'

for i in tqdm(range(len(x_train))):
  out = ""
  #x_train[i] = araby.strip_tashkeel(x_train[i])
  for c in x_train[i]:
    if c not in execluded:
      out += c
  x_train[i] = out

for i in tqdm(range(len(x_test))):
  out = ""
  #x_test[i] = araby.strip_tashkeel(x_test[i])
  for c in x_test[i]:
    if c not in execluded:
      out += c
  x_test[i] = out
## Almost 1900 examples are taken to validate the model to make sure the model generalizes well
X_train, X_valid , y_train, y_valid = train_test_split(x_train, y_train, test_size = 0.1, random_state = 41)
# Creating a mapping from unique characters to indices
vocab = set(' '.join(x_train))
char2idx = {u:i+1 for i, u in enumerate(vocab)}

def to_sequences(X):
  X = [[char2idx[char] for char in line] for line in X]
  X = pad_sequences(X, padding='post', value=0, maxlen = 100)
  return X
 
X_train = to_sequences(X_train)
X_valid = to_sequences(X_valid)

y_train = np.array(y_train)
y_valid = np.array(y_valid)
"""
## My model
"""
## Steps taken to get the 95% accuracy:

### Training 40 epochs on the first Model, then realoading the same weights 
### following up with 20 epochs on the second model which only differs from
### the first in the dropout layers

from tensorflow import keras


############### 70 epochs #########

model = Sequential()
model.add(Input((100,)))
model.add(Embedding(len(char2idx)+1,256))
model.add(Bidirectional(tf.keras.layers.LSTM(units = 256, return_sequences=True)))
model.add(Dropout(0.7))
model.add(Bidirectional(tf.keras.layers.LSTM(units = 1024)))
model.add(Dropout(0.75)) 
model.add(Dense((512), activation = 'relu'))
model.add(Dropout(0.75)) 
model.add(Dense(len(label2name), activation = 'softmax'))
model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
"""
## Model summary
"""
model.summary()
"""
## Training process
"""
## Creating a callback to recall best accuracy achieved on the validation set

callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, min_delta=0.0001, min_lr=0.0001)]
callbacks += [tf.keras.callbacks.ModelCheckpoint('full_verse.h5', monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')]
## Train the model for than 60 epochs at least
model.fit(X_train, y_train, validation_data= (X_valid, y_valid), epochs = 70, batch_size= 32, shuffle = True, callbacks=callbacks)
"""
## after modelig
"""
## Loading best weights that achieved best accuracy

model = tf.keras.models.load_model('full_verse.h5')
"""
## Classify on the test set splitted from the dataset 
"""
def classify(sentence):

  sequence = [char2idx[char] for char in sentence]
  sequence = pad_sequences([sequence], maxlen = X_train.shape[1], padding='post', value=0)

  pred = model.predict(sequence)[0]
  pred = np.argmax(pred, 0)
  return pred
prediction = []
for i in tqdm(range(len(x_test))):
    prediction.append(classify(x_test[i]))
from sklearn.metrics import classification_report
print(classification_report(y_test, prediction))
"""
## Submission
Submit the predictions on the qualifying leaderboard
"""
submission = pd.read_csv('test.csv')
submission
test = np.array(submission['data'])
test
for i in tqdm(range(len(test))):
  out = ""
  #test[i] = araby.strip_tashkeel(test[i])
  for c in test[i]:
    if c not in execluded:
      out += c
  test[i] = out
preds = []
for i in tqdm(range(len(test))):
    preds.append(classify(test[i]))
ids = [str(id_val) for id_val in range(submission.shape[0])]
sample_submission = pd.DataFrame(np.array([ids,preds]).T,columns=['id','labels'])
sample_submission.to_csv("sample_submission13_95xxx.csv",index=False)
sample_submission
"""
## Final submission
Submit the predictions on the final qualifying leaderboard
"""
submission_final = pd.read_csv('/content/final_round_test.csv')
submission_final
final_test = np.array(submission_final['data'])
final_test
for i in tqdm(range(len(final_test))):
  out = ""
  #test[i] = araby.strip_tashkeel(test[i])
  for c in final_test[i]:
    if c not in execluded:
      out += c
  final_test[i] = out
final_preds = []
for i in tqdm(range(len(final_test))):
    final_preds.append(classify(final_test[i]))
ids = [str(id_val) for id_val in range(submission_final.shape[0])]
sample_submission = pd.DataFrame(np.array([ids,final_preds]).T,columns=['id','labels'])
sample_submission.to_csv("final_submission_v9530.csv",index=False)
sample_submission
## Concatenate  to get ' High results '
## Concatenate the training data with the predictions made in order to get a higher generalization of the process
X_test = to_sequences(x_test)
TEST = to_sequences(test)
Preds = np.array(preds)
Y_test = np.array(y_test)
X_total = np.concatenate((X_train,X_test,TEST),axis=0)
y_total = np.concatenate((y_train,Y_test,Preds),axis=0)

callbacks2 = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, min_delta=0.0001, min_lr=0.0001)]
callbacks2 += [tf.keras.callbacks.ModelCheckpoint('full_verse2.h5', monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')]
## setting 6 to 10 epochs is enough
model.fit(X_total, y_total, validation_data= (X_valid, y_valid), epochs =8 , batch_size= 32, shuffle = True,callbacks= callbacks2)
model = tf.keras.models.load_model('full_verse2.h5')

## After that you can re-create your submission results in order to get a better score on the leaderboard