import keras

keras.__version__

from keras.layers import Embedding

# Number of maximum tokens is equal of maximum word index + 1
max_number_of_tokens = 1000
embedding_dimentionality = 64
embedding_layer = Embedding(max_number_of_tokens, embedding_dimentionality)

"""
Explanation: Chapter 6.1.2 - Using word embeddings
Embedding layer with Keras
End of explanation
"""


from keras.datasets import imdb
from keras.preprocessing.sequence import pad_sequences

# Number of words considered as features
max_features = 10000

# Cutting reviews after only 20 words
sequence_max_length = 20

# Loading data
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words = max_features)

x_train.shape

x_train_sequence = pad_sequences(x_train, maxlen = sequence_max_length)

x_train_sequence.shape

x_train[0:2]

x_train[0].__getitem__(-20)

x_train_sequence[0, :]

x_train_sequence[0]

x_train_sequence[1]

"""
Explanation: The layers transforms a 2D input tensor of integer of shape (number_of samples, sequence_length) into a 3D floating point tensor, of shape (number_of_samples, sequence_length, embedding_dimensionality.)
Such tensor can be processed a RNN layer of a 1D convolutional layer.
IMDB example
End of explanation
"""


from keras.models import Sequential
from keras.layers import Flatten, Dense

model = Sequential()
model.add(Embedding(input_dim = max_features, output_dim = 8, input_length = sequence_max_length))
model.add(Flatten())
model.add(Dense(units = 1, activation = 'sigmoid'))

# Compiling the model
model.compile(optimizer = 'rmsprop', 
              loss = 'binary_crossentropy', 
              metrics = ['acc'])

model.summary()

# Training
history = model.fit(x = x_train_sequence, 
                    y = y_train, 
                    epochs = 10, 
                    batch_size = 32, 
                    validation_split = 0.2)

"""
Explanation: Model
End of explanation
"""


import os

imdb_dir = './data/Chapter 6.1.2 - Using word embeddings/aclImdb/'

train_dir = os.path.join(imdb_dir, 'train')

labels = []
texts = []

for label_type in ['neg', 'pos']:
    dir_name = os.path.join(train_dir, label_type)
    for fname in os.listdir(dir_name):
        # Taking into consideration files which are only .txt
        if fname[-4:] == '.txt':
            f = open(os.path.join(dir_name, fname), encoding="utf8")
            texts.append(f.read())
            f.close()
            if label_type == 'neg':
                labels.append(0)
            else:
                labels.append(1)

len(labels)

len(texts)

texts[0]

labels[0]

"""
Explanation: Using pre-trained word embeddings
The data can be downloaded from: http://mng.bz/0tIo
End of explanation
"""


from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import numpy as np

# Using only first 100 words of each review
maxlen = 100

# Number of training samples
training_samples = 200

# Number of validation samples
validation_samples = 10000

# Tokenizing only top 10 000 words in the dataset.
max_words = 10000 

# Initializing Tokenizer
tokenizer = Tokenizer(num_words = max_words)

# Fitting the Tokenizer on the text
tokenizer.fit_on_texts(texts)

# Text to sequence
sequences = tokenizer.texts_to_sequences(texts)

sequences[0:2]

# Word index
word_index = tokenizer.word_index

type(word_index)

first10pairs = {k: word_index[k] for k in list(word_index)[:10]}

first10pairs

# Padding the sequence
data = pad_sequences(sequences, maxlen = maxlen)

data.shape

labels = np.asarray(labels)

labels.shape

indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]

# Splitting the data into train and validation datasets
x_train = data[:training_samples]
y_train = labels[:training_samples]
x_val = data[training_samples: training_samples + validation_samples]
y_val = labels[training_samples: training_samples + validation_samples]

x_train.shape

x_val.shape

"""
Explanation: Tokenizing the data
End of explanation
"""


# Importing tqdm to show a progress bar
from tqdm import tqdm

glove_dir = './data/Chapter 6.1.2 - Using word embeddings/glove.6B/'
embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), 
         encoding = 'utf-8')
for line in tqdm(f):
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], 
                       dtype = 'float32')
    embeddings_index[word] = coefs
f.close()

len(embeddings_index)

embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))

for word, i in word_index.items():
    if i < max_words:
        embedding_vector = embeddings_index.get(word)
        # Words not found in the embedding index will be represented as zeros
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector

embedding_matrix

"""
Explanation: GloVe Embedding
Download from: http://nlp.stanford.edu/data/glove.6B.zip
End of explanation
"""


from keras.models import Sequential
from keras.layers import Embedding, Flatten, Dense

model = Sequential()
model.add(Embedding(input_dim = max_words, 
                    output_dim = embedding_dim, 
                    input_length = maxlen))
model.add(Flatten())
model.add(Dense(units = 32, 
                activation = 'relu'))
model.add(Dense(units = 1, 
                activation = 'sigmoid'))
model.summary()

# Loading pretrained word embeddings
model.layers[0].set_weights([embedding_matrix])
# Freezing the layer
model.layers[0].trainable = False

model.compile(optimizer = 'rmsprop',
              loss = 'binary_crossentropy',
              metrics = ['acc'])

history = model.fit(x = x_train, 
                    y = y_train,
                    epochs = 10,
                    batch_size = 32,
                    validation_data = (x_val, y_val))

model.save_weights('./saved_checkpoints/Chapter 6.1.2 - Using word embeddings/pre_trained_glove_model.h5')

"""
Explanation: Model
End of explanation
"""


import matplotlib.pyplot as plt

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

"""
Explanation: Performance
End of explanation
"""


from keras.models import Sequential
from keras.layers import Embedding, Flatten, Dense

model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=maxlen))
model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.summary()

model.compile(optimizer='rmsprop',
              loss='binary_crossentropy',
              metrics=['acc'])
history = model.fit(x_train, y_train,
                    epochs=10,
                    batch_size=32,
                    validation_data=(x_val, y_val))

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

"""
Explanation: The model overfits very quickly.
Model without pre-trained embeddings
End of explanation
"""


