# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Text Classification with gensim & word2vec

# +
import numpy as np
import pandas as pd
import tensorflow as tf
import pickle

from sklearn.preprocessing import MultiLabelBinarizer
from keras.layers import Input, Dense
from keras.models import Model
from keras.callbacks import TensorBoard
from gensim.models import KeyedVectors 
from keras.utils import Sequence

print(tf.__version__)
# -

# Download the data from GCS
# !wget 'https://storage.googleapis.com/movies_data/movies_metadata.csv'

data = pd.read_csv('movies_metadata.csv')
data.head()

# +
# urllib.request.urlretrieve('https://storage.googleapis.com/bq-imports/descriptions.p', 'descriptions.p')
# urllib.request.urlretrieve('https://storage.googleapis.com/bq-imports/genres.p', 'genres.p')

descriptions = pickle.load(open('pickle/descriptions.p', 'rb'))
genres = pickle.load(open('pickle/genres.p', 'rb'))



# +
train_size = int(len(descriptions) * .8)

train_descriptions = descriptions[:train_size].astype('str')
train_genres = genres[:train_size]

test_descriptions = descriptions[train_size:].astype('str')
test_genres = genres[train_size:]

# +
encoder = MultiLabelBinarizer()
encoder.fit_transform(train_genres)
train_encoded = encoder.transform(train_genres)
test_encoded = encoder.transform(test_genres)
num_classes = len(encoder.classes_)

print(encoder.classes_)
print(train_descriptions.values[0])
print(train_encoded[0])


# +
# Initialize word2vec model
# %%time

word2vec = KeyedVectors.load_word2vec_format("./models/GoogleNews-vectors-negative300.bin", binary=True)

# +
# preprocess for OOV

for i in range(len(train_descriptions)):
    if train_descriptions.values[i] == ' ':
        train_descriptions.values[i] = 'nan'

for i in range(len(test_descriptions)):
    if test_descriptions.values[i] == ' ':
        test_descriptions.values[i] = 'nan'
        
# convert OOV words to 'nan'

for i in range(len(train_descriptions)):
    text = train_descriptions.values[i]
    text_list = text.split(' ')

    for j in range(len(text_list)):
        if text_list[j] not in word2vec.vocab:
            text_list[j] = 'nan'
    
    train_descriptions.values[i] = ' '.join(text_list)


# -

# Sequence of data
class BatchSequence(Sequence):
    
    def __init__(self, x_set, y_set, batch_size, shuffle=True):
        data_size = len(x_set)

        if shuffle:
            shuffle_indices = np.random.permutation(np.arange(data_size))
            self.x = x_set[shuffle_indices]
            self.y = y_set[shuffle_indices]
        else:
            self.x, self.y = x_set, y_set
            
        self.batch_size = batch_size

    def __len__(self):
        return int(np.ceil(len(self.x) / float(self.batch_size)))

    def __getitem__(self, idx):
        start_index = idx * self.batch_size
        end_index = min((idx + 1) * self.batch_size, len(self.x))
        
        batch_x = self.x[start_index: end_index]
        batch_y = self.y[start_index: end_index]

        splited_data = list(map(lambda x: x.split(' '), batch_x))
        query = list(map(lambda a: list(filter(lambda b: b in word2vec.vocab, a)), splited_data))
        embeddings = np.array(list(map(lambda a: np.mean(list(map(lambda b: word2vec.wv[b], a)), axis=0), query)))
        
        X, y = np.array(embeddings), np.array(batch_y)

        return X, y



# +
# word2vec embedding dimension
wor2vec_dim = 300

# Input Layers
word_input = Input(shape=(wor2vec_dim, ), dtype=tf.float32)  # (batch_size, sent_length)

# Hidden Layers
x = Dense(64, activation='relu')(word_input)

# Output Layer
predict = Dense(units=9, activation='sigmoid')(x)

model = Model(inputs=[word_input], outputs=predict)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

model.summary()


# +
# Create an instance of batch sequence
batch_size = 32

train_batchSequence = BatchSequence(train_descriptions.values, train_encoded, batch_size)
valid_batchSequence = BatchSequence(test_descriptions.values, test_encoded, batch_size)


# +
# Train model

logfile_path = './log'
tb_cb = TensorBoard(log_dir=logfile_path, histogram_freq=0)

history = model.fit_generator(train_batchSequence,
                              epochs=5, 
                              validation_data=valid_batchSequence,
                              callbacks=[tb_cb])

# +
# Evaluate model

score = model.evaluate_generator(valid_batchSequence)
print('Test score:', score[0])
print('Test accuracy;', score[1])

# -



StarcoderdataJupyterScriptsDedupFiltered