# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Tutorial on Keras with Gensim
# https://www.depends-on-the-definition.com/guide-to-word-vectors-with-gensim-and-keras/
#
# - http://www.orbifold.net/default/2017/01/10/embedding-and-tokenizer-in-keras/
# - https://github.com/keras-team/keras/issues/853
# - http://adventuresinmachinelearning.com/gensim-word2vec-tutorial/
# - https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/
# - https://stats.stackexchange.com/questions/320701/how-to-use-keras-pre-trained-embedding-layer
# - https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html
# - https://codekansas.github.io/blog/2016/gensim.html

# +
% load_ext autoreload
% autoreload 2

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

from nltk.tokenize import WordPunctTokenizer
from collections import Counter

from keras.layers import Dense, Input, LSTM, CuDNNLSTM, Embedding, Dropout,SpatialDropout1D, Bidirectional
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers.normalization import BatchNormalization
from keras.utils import to_categorical

# Visualization
import matplotlib.pyplot as plt
import matplotlib.cm as cmap
# %matplotlib inline

import utils
import text_utils

# +
# Constants

OUTPUT_DIR = './week-7-plots'

SRC_ENHANCED_COMMENTS = '../data/pol/comments-root-all-pol-enhanced.csv'

SRC_GENSIM_EMBEDDING = '../data/embedding/gensim-guardian-comments-50-tokenized.bin'
# -

# #### Load embedding

# +
word_vectors = text_utils.load_embedding()
index2word = word_vectors.index2word  # Map index to word
word2index = dict((x, y) for y,x in enumerate(word_vectors.index2word))  # Map word to index

print("Number of word vectors: {}".format(len(word_vectors.vocab)))
# -

word_vectors.most_similar_cosmul(positive=['woman', 'king'], negative=['man'])

# #### Preprocess data

# +
EMBEDDING_DIM = 50
MAX_SEQUENCE_LENGTH = 300

comments = pd.read_csv(SRC_ENHANCED_COMMENTS)
X = comments['comment_text']
y = to_categorical(comments['bin'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
# Pad/Cut tokenized comments to a certain length
X_train, X_test = text_utils.pad_or_cut_tokenized_comments(X_train, X_test, word2index, MAX_SEQUENCE_LENGTH)
# -

# #### Prepare Keras Model

model = Sequential()
model.add(Embedding(len(word_vectors.vocab),
                    EMBEDDING_DIM,
                    mask_zero=False,
                    weights=[word_vectors.syn0],
                    input_length=MAX_SEQUENCE_LENGTH,
                    trainable=False))
model.add(SpatialDropout1D(0.2))
# model.add(LSTM(64, return_sequences=True, recurrent_dropout=0.3, dropout=0.3))
model.add(LSTM(32, recurrent_dropout=0, dropout=0.95))
model.add(Dense(2, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = ['binary_accuracy'])

model.summary()

# +
epochs = 10
batch_size = 32

# Here we train the Network.
try:
    history = model.fit([X_train], y_train, validation_split=0.1,
                        batch_size = batch_size, epochs = epochs,
                        verbose = 2, shuffle=True)
except KeyboardInterrupt:
    print("Fitting stopped manually")
# -

text_utils.inspect_preprocessed_comment(comments, X_train, index2word, 0)

utils.plot_history(history)

# +
# Measuring score and accuracy on test set

score, acc = model.evaluate([X_test], y_test, verbose = 2,
                            batch_size = batch_size)
print("Logloss score: %.2f" % (score))
print("Test set Accuracy: %.2f" % (acc))
# -

plt.hist(comments['comment_text'].str.split().len)

StarcoderdataJupyterScriptsDedupFiltered