# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <hr style="height:2px;border:none"/>

# <center>
#     <img src="http://sct.inf.utfsm.cl/wp-content/uploads/2020/04/logo_di.png" style="width:60%">
#     <h1> INF-395/477 Redes Neuronales Artificiales I-2020 </h1>
#     <h3>  Tarea 2 - Redes Neuronales Convolucionales y Recurrentes  </h3>
# </center>
#
# Nombres: <NAME>  & <NAME>
#
# Roles: 201573598-0 &201573573-5
#
# Correos:
# <EMAIL> & <EMAIL>
# <hr style="height:2px;border:none"/>
#
#
# **Temas**  
#
# * Entrenamiento de Redes Neuronales Profundas. 
# * Modelos de Auto-Encoder
# * Redes Convolucionales y Recurrentes. 
#
# <hr style="height:2px;border:none"/>

# # Parte 1 - Tema Libre

# ## Tema
# > Tema 13. No tiene sentido usar una red convolucional para aprendizaje de secuencias, su error será siempre mayor que el de una red recurrente.

# + [markdown] colab_type="text" id="MC0aIMdBsy8F"
# ## Variables generales y de entorno

# + colab={} colab_type="code" id="P16ZFRDq7WVE"
from __future__ import print_function

import warnings
warnings.filterwarnings('ignore')

import os
import numpy as np
import pandas as pd
import sys
import seaborn as sns
import matplotlib.pyplot as plt
from random import randint

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

from keras import backend as K
from keras.preprocessing import sequence
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Activation, Flatten, Embedding, LSTM, Conv1D, MaxPooling1D, Input, GlobalMaxPooling1D
from keras.utils import to_categorical
from keras.initializers import Constant

from keras.datasets import imdb

import nltk
nltk.download("brown")
from nltk.corpus import brown
brown.words(categories='news')


# + colab={} colab_type="code" id="u6vKm2e8yi5T"
# ! mkdir data_in

# + colab={} colab_type="code" id="MiwR3Q4ayjCP"
# ! pip install kaggle

# + colab={} colab_type="code" id="x0uZfsxuyq7N"
# ! mkdir ~/.kaggle
# ! touch ~/.kaggle/kaggle.json
# ! echo '{"username":"verack","key":"adbc9010cabd6ada149effbea0960f9b"}' > ~/.kaggle/kaggle.json
# ! chmod 600 ~/.kaggle/kaggle.json

# + colab={} colab_type="code" id="9qWnkeWryrRk"
# ! rm -rf data_in/*

# + [markdown] colab_type="text" id="PI3JZvtcUxNh"
# Descarga de los vectores de gloVe (100)
# Link: https://www.kaggle.com/rtatman/glove-global-vectors-for-word-representation

# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="byziI6vMUwJR" outputId="af04d4ec-5aa8-46b3-a721-0b6ddbc89423"
# ! mkdir data_in/glove
# ! kaggle datasets download rtatman/glove-global-vectors-for-word-representation -p data_in/glove

# + colab={"base_uri": "https://localhost:8080/", "height": 91} colab_type="code" id="ffD4sE1QWQbC" outputId="a60ceec1-40e1-4421-c0bf-8c8e3e935a73"
# ! unzip data_in/glove/glove-global-vectors-for-word-representation.zip -d data_in/glove

# + [markdown] colab_type="text" id="yMuiy0W8Gmx-"
# Metricas Globales

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="CKg-UgjqGp6f" outputId="a4eb0834-3a36-4b1a-e039-1e5a17e68546"
def f1(y_true, y_pred):
    def recall(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

    def precision(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision
    precision = precision(y_true, y_pred)
    recall = recall(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))


# + [markdown] colab_type="text" id="YqpuoG4gf4sC"
# ### Comparación de Redes CNN y RNN para el caso del analisis de sentimientos para las reviews de IMDB

# + [markdown] colab_type="text" id="4UWieasflStK"
# Link: https://keras.io/api/datasets/imdb/

# + [markdown] colab_type="text" id="xxNrZG3YV4f6"
# #### Preprocesamiento de los datos

# + colab={} colab_type="code" id="tT8IjKqcbn7Y"
# Embedding
max_features = 20000
maxlen = 100
embedding_size = 128

# Convolution
kernel_size = 5
filters = 64
pool_size = 4

# LSTM
lstm_output_size = 70

# Training
batch_size = 30
epochs = 2

# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="JvmrWyhrkb6d" outputId="4c0f611c-e8e0-447b-91f9-ac48447ad785"
'''
Note:
batch_size is highly sensitive.
Only 2 epochs are needed as the dataset is very small.
'''

print('Loading data...')
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="qaXygbjXkcry" outputId="7e1abdb8-661c-4225-a3cd-d147a5003a8d"
#Calculo del promedio
suma = 0
for i in range(25000):
  suma = len(x_train[i]) + suma
print("El promedio de palabras para data_train",suma/25000)

suma = 0
for i in range(25000):
  suma = len(x_test[i]) + suma
print("El promedio de palabras para data_test",suma/25000)

#average = sum(len(word) for word in words) / len(words)

# + colab={"base_uri": "https://localhost:8080/", "height": 128} colab_type="code" id="yaQ1mUdEb1ok" outputId="cc0b7da7-122a-4d12-c608-cf180a8f1021"
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)

# + [markdown] colab_type="text" id="3G1dQ29RcXKE"
# modelo CNN

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="Cl1CN447b6tQ" outputId="07e5f133-d2dc-44b7-ff25-e17a505a96b3"
print('Build CNN model...')

model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Conv1D(filters,
                 kernel_size,
                 padding='valid',
                 activation='relu',
                 strides=1))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(Flatten())
#model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# + colab={"base_uri": "https://localhost:8080/", "height": 421} colab_type="code" id="2qF1_nKpe6gM" outputId="be753f48-750b-4183-e083-dafa45ee8ce3"
model.summary()

# + colab={"base_uri": "https://localhost:8080/", "height": 182} colab_type="code" id="63KolKWLb6xx" outputId="21a42bae-95ea-423f-ffd3-56fc4f6a8b3f"
print('Train CNN model...')
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)

# + [markdown] colab_type="text" id="Btaf87ekcUAS"
# modelo RNN

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="lo7BLSNEcSt4" outputId="363b94ee-0714-4bfb-c770-37e340335185"
print('Build RNN model...')

model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
#model.add(Conv1D(filters,
#                 kernel_size,
#                 padding='valid',
#                 activation='relu',
#                 strides=1))
#model.add(MaxPooling1D(pool_size=pool_size))
model.add(LSTM(lstm_output_size,dropout=0.2,recurrent_dropout=0.2))
model.add(Dense(lstm_output_size,activation="relu"))
model.add(Dense(1,activation="sigmoid"))

#model.add(Dense(1))
#model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# + colab={"base_uri": "https://localhost:8080/", "height": 182} colab_type="code" id="Z8rlUichcfNR" outputId="73001959-9bbd-4304-883c-bbf5ab4aa2ef"
print('Train RNN model...')
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)

# + [markdown] colab_type="text" id="bVFg_sRj3THN"
# ### Comparación de Redes CNN y RNN para el caso de clasificación del tipo de noticias

# + [markdown] colab_type="text" id="Gei9u4WiVr4v"
# Descarga del dataset Link: https://www.kaggle.com/wunderbarx/20-newsgroup

# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="gPq1pu2Lc3Bq" outputId="e35d1211-9003-45ff-f454-f9756c0d1b51"
# ! mkdir data_in/20_news
# ! kaggle datasets download wunderbarx/20-newsgroup -p data_in/20_news

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" id="xVi-ARaA8oW9" outputId="70428fda-eecd-4d89-c2f4-a8b04244c1c0"
# ! unzip data_in/20_news/20-newsgroup.zip -d data_in/20_news

# + [markdown] colab_type="text" id="dfvGNzEt9q9p"
# Preprocesamiento de los datos

# + [markdown] colab_type="text" id="mASIapDN-qPa"
# Variables globales

# + colab={} colab_type="code" id="PAhzk0Dg95QV"
BASE_DIR = 'data_in/'
GLOVE_DIR = os.path.join('data_in/', 'glove')
TEXT_DATA_DIR = os.path.join('data_in/20_news', '20_newsgroups')
MAX_SEQUENCE_LENGTH = 1000
MAX_NUM_WORDS = 20000
EMBEDDING_DIM = 100
VALIDATION_SPLIT = 0.2

# + [markdown] colab_type="text" id="-zhuWtP0-yw6"
# Configuración del vector de embedding

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="f5TYUraI-tGK" outputId="4d1a7335-053d-4be5-8bea-4479a75e6980"
# first, build index mapping words in the embeddings set
# to their embedding vector

print('Indexing word vectors.')

embeddings_index = {}
with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt')) as f:
    for line in f:
        word, coefs = line.split(maxsplit=1)
        coefs = np.fromstring(coefs, 'f', sep=' ')
        embeddings_index[word] = coefs

print('Found %s word vectors.' % len(embeddings_index))

# + [markdown] colab_type="text" id="OYuATcbI_vUA"
# Se procesa el dataset de textos

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="f0BLHWoN_uhL" outputId="a7104598-ee26-4861-84bd-f40966a1bad3"
# second, prepare text samples and their labels
print('Processing text dataset')

texts = []  # list of text samples
labels_index = {}  # dictionary mapping label name to numeric id
labels = []  # list of label ids
for name in sorted(os.listdir(TEXT_DATA_DIR)):
    path = os.path.join(TEXT_DATA_DIR, name)
    if os.path.isdir(path):
        label_id = len(labels_index)
        labels_index[name] = label_id
        for fname in sorted(os.listdir(path)):
            if fname.isdigit():
                fpath = os.path.join(path, fname)
                args = {} if sys.version_info < (3,) else {'encoding': 'latin-1'}
                with open(fpath, **args) as f:
                    t = f.read()
                    i = t.find('\n\n')  # skip header
                    if 0 < i:
                        t = t[i:]
                    texts.append(t)
                labels.append(label_id)

print('Found %s texts.' % len(texts))

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="HULHTPCCdgsT" outputId="cc48f249-29b3-4f3e-80ac-bb796a391453"
suma = 0
for texto in texts:
  suma = suma + len(texto)
promedio = suma/len(texts)
print(promedio)

# + [markdown] colab_type="text" id="cWx6h3nSAG9f"
# Se vectorizan los inputs de texto a un vector 2D

# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="lpAB_TtlAHVN" outputId="25720905-1c2c-4f95-d5a8-ea2767eedc96"
# finally, vectorize the text samples into a 2D integer tensor
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

# + [markdown] colab_type="text" id="ih2vcDmkAfe6"
# Se separan los datos de train y de test

# + colab={} colab_type="code" id="NlZAUn2tAe9E"
# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])

x_train = data[:-num_validation_samples]
y_train = labels[:-num_validation_samples]
x_val = data[-num_validation_samples:]
y_val = labels[-num_validation_samples:]

# + [markdown] colab_type="text" id="n5X4OFT8A3Rq"
# Se prepara la matriz de embedding

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="pGXBmYoNA3j7" outputId="82236f17-a758-4619-be57-ea892680889a"
print('Preparing embedding matrix.')

# prepare embedding matrix
num_words = min(MAX_NUM_WORDS, len(word_index) + 1)
embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
for word, i in word_index.items():
    if i >= MAX_NUM_WORDS:
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

# + [markdown] colab_type="text" id="Vix6YaBTBCdt"
# Se prepara la capa de embedding

# + colab={} colab_type="code" id="zqLVGumcBC5U"
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
                            EMBEDDING_DIM,
                            embeddings_initializer=Constant(embedding_matrix),
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=False)

# + [markdown] colab_type="text" id="8skpdBo7BN7n"
# Se entrena la red convolucional 1D

# + colab={"base_uri": "https://localhost:8080/", "height": 264} colab_type="code" id="_Iq2a2m4BOZ7" outputId="534a1ea7-7352-458b-b9d1-78eda447200a"
print('Training model.')

# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
#x = MaxPooling1D(5)(x)
#x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)

model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])


# + colab={"base_uri": "https://localhost:8080/", "height": 531} colab_type="code" id="6Cw9iVq_C2c1" outputId="7ee423f7-3694-4c76-f094-910025c5f5cd"
model.summary()

# + colab={"base_uri": "https://localhost:8080/", "height": 201} colab_type="code" id="nkIoOZjpCFTg" outputId="72a72e7e-c6b2-4d92-a416-dcaa7d6ae97a"
# Training
batch_size = 100
epochs = 3

model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_val, y_val))

score, acc = model.evaluate(x_val, y_val, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)

# + [markdown] colab_type="text" id="hsR8TmEOGzck"
# Metricas para la red CNN

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" id="g4cSTcZVG2AD" outputId="e89a3baf-323b-4053-a998-7f5ec824433a"
fig = plt.figure(figsize=(20, 20))
y_pred = model.predict(x_val)
Y_pred = np.argmax(y_pred, 1)
Y_test = np.argmax(y_val, 1)
mat = confusion_matrix(Y_test, Y_pred)
sns.heatmap(mat.T, square=True, annot=True, cbar=False, cmap=plt.cm.Blues)
plt.xlabel('Predicted Values')
plt.ylabel('True Values');
plt.show();

# + [markdown] colab_type="text" id="M6tCK9LkHUwK"
# Tabla de rendimientos para la CNN

# + colab={"base_uri": "https://localhost:8080/", "height": 513} colab_type="code" id="5XR4acszHfbl" outputId="dd7af019-fa81-4eaa-8593-d19650d50c40"
print(classification_report(Y_test, Y_pred))

# + [markdown] colab_type="text" id="PFWubZ_WDkJL"
# Red LSTM

# + colab={} colab_type="code" id="YgSTJBK2IDbd"
# LSTM
lstm_output_size = 100

sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)

x = LSTM(lstm_output_size)(embedded_sequences)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

# + colab={"base_uri": "https://localhost:8080/", "height": 201} colab_type="code" id="-iTQWGegSdWO" outputId="7f8ec8ca-1d09-4448-d186-b64bc52f30bc"
# Training
batch_size = 100
epochs = 3

model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_val, y_val))

score, acc = model.evaluate(x_val, y_val, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" id="drfsAlimSk6A" outputId="319bb665-dc26-44f5-98a9-d4a004c33f53"
fig = plt.figure(figsize=(20, 20))
y_pred = model.predict(x_val)
Y_pred = np.argmax(y_pred, 1)
Y_test = np.argmax(y_val, 1)
mat = confusion_matrix(Y_test, Y_pred)
sns.heatmap(mat.T, square=True, annot=True, cbar=False, cmap=plt.cm.Blues)
plt.xlabel('Predicted Values')
plt.ylabel('True Values');
plt.show();

# + colab={"base_uri": "https://localhost:8080/", "height": 513} colab_type="code" id="VstzF6qTS8xZ" outputId="703dc394-d0eb-42fe-fe1e-560e9f17186f"
print(classification_report(Y_test, Y_pred))

# + [markdown] colab_type="text" id="1xKzQIfuySwU"
# ## datasets sinteticos
#
#

# + [markdown] colab_type="text" id="e5xR9l1MmDFS"
# Link: https://keras.io/api/datasets/imdb/

# + [markdown] colab_type="text" id="iZM1YbtQsvFK"
# Sentiment Analisys with CNN

# + colab={} colab_type="code" id="teg80FAosvFg"
# Embedding
max_features = 25000
parcial_len = 80
maxlen = 100
difference = maxlen - parcial_len
embedding_size = 128


# Convolution
kernel_size = 5
filters = 64
pool_size = 4

# LSTM
lstm_output_size = 70

# Training
batch_size = 30
epochs = 2

# + [markdown] colab_type="text" id="zO50HVv-378j"
# Generación de Sustantivos

# + colab={"base_uri": "https://localhost:8080/", "height": 74} colab_type="code" id="G4mAmA814A8Z" outputId="d85ae379-753e-4a82-8a32-abf9f6276180"
brown_news_tagged = brown.tagged_words(categories='news')
#tag_fd = nltk.FreqDist(tag for (word, tag) in brown_news_tagged)
#tag_fd.keys()
lista_sustantivos = []
i = 1000 # Se crea una lista de 1000 sustantivos
for (word, tag) in brown_news_tagged:
  if tag == 'NN' and i > 0:
    lista_sustantivos.append(word.lower())
    i = i - 1

print(lista_sustantivos)
print(len(lista_sustantivos))

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="CnkfZ2a03-y9" outputId="b8093cec-6e4b-41cd-f610-9b975fe64601"
word_to_id = imdb.get_word_index()

INDEX_FROM=3   # word index offset

word_to_id = {k:(v+INDEX_FROM) for k,v in word_to_id.items()}
word_to_id["<PAD>"] = 0
word_to_id["<START>"] = 1
word_to_id["<UNK>"] = 2
word_to_id["<UNUSED>"] = 3

id_to_word = {value:key for key,value in word_to_id.items()}
#print(' '.join(id_to_word[id] for id in train_x[0] ))


id_lista_sustantivos = []

for word in lista_sustantivos:
  try:
    id_lista_sustantivos.append(word_to_id[word])
  except KeyError:
    continue


len(id_lista_sustantivos)

# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="qX8hDOz-tmTK" outputId="d3e0153e-ec4c-47dd-e031-36fc93f98a89"
'''
Note:
batch_size is highly sensitive.
Only 2 epochs are needed as the dataset is very small.
'''

print('Loading data...')
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')

# + colab={"base_uri": "https://localhost:8080/", "height": 55} colab_type="code" id="xslR9rgFtnLG" outputId="e3d2d837-afe2-4567-c91d-87999fcdc408"
print(x_train[0])

# + colab={"base_uri": "https://localhost:8080/", "height": 55} colab_type="code" id="VAwEHxrsmyd_" outputId="c75b5b65-106f-45c2-c5dd-7206cfbf6e79"
print(x_test[0])

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" id="a7eGM2wQgGyt" outputId="c06f6972-82f1-462f-9bf0-e283d2a7fb20"
#Calculo del promedio
suma = 0
for i in range(25000):
  suma = len(x_train[i]) + suma
print("El promedio de palabras para data_train",suma/25000)

suma = 0
for i in range(25000):
  suma = len(x_test[i]) + suma
print("El promedio de palabras para data_test",suma/25000)

#average = sum(len(word) for word in words) / len(words)

# + colab={} colab_type="code" id="6VUmrtFpHyO3"
def agregar_palabras_random(dataset, lista_palabras_random, cant_palabras):
  length = len(lista_palabras_random)
  for j in range(len(dataset)):
    if len(dataset[j]) < 5:
        continue
    for i in range(cant_palabras-10):
      rand_index = randint(2, len(dataset[j])-5)
      rand_sust = randint(0, length-1)
      dataset[j].insert(rand_index, lista_palabras_random[rand_sust])
  return dataset


# + colab={"base_uri": "https://localhost:8080/", "height": 74} colab_type="code" id="k5deKkoDGgTL" outputId="1f2f940b-ace6-4f03-e1c0-e9dcfd5ae8c7"
s = " "

lista1 = []

oracion = x_train[0]
for i in oracion:
  lista1.append(id_to_word[i])

print(s.join(lista1))
#print(x_train[0])

i=0

x_train = agregar_palabras_random(x_train, id_lista_sustantivos, difference)
x_test = agregar_palabras_random(x_test, id_lista_sustantivos, difference)


lista2 = []
oracion = x_train[0]
for i in oracion:
  lista2.append(id_to_word[i])

print(s.join(lista2))
#print(x_train[0])


# + colab={"base_uri": "https://localhost:8080/", "height": 72} colab_type="code" id="jpyq12QIsvFp" outputId="f62a3bb5-918e-4bde-81cf-f7a9766b2f42"
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)

# + [markdown] colab_type="text" id="pgUxnD_NsvFz"
# modelo CNN

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="sKsBIvXBsvF2" outputId="7dfac933-5303-4a5d-9d25-1a2e38bf9ac8"
print('Build CNN model...')

model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Conv1D(filters,
                 kernel_size,
                 padding='valid',
                 activation='relu',
                 strides=1))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(Flatten())
#model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# + colab={"base_uri": "https://localhost:8080/", "height": 421} colab_type="code" id="GuFj_LuDsvGC" outputId="07adea1c-7809-4d38-d25d-a1cf1816be65"
model.summary()

# + colab={"base_uri": "https://localhost:8080/", "height": 239} colab_type="code" id="R74tqYQHsvGJ" outputId="3bceece1-8b48-4cbb-94a8-f0cbde33241a"
print('Train CNN model...')
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)

# + [markdown] colab_type="text" id="M_5Jh7S8svGS"
# modelo RNN

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="MCd_l6v6svGU" outputId="363b94ee-0714-4bfb-c770-37e340335185"
print('Build RNN model...')

model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
#model.add(Conv1D(filters,
#                 kernel_size,
#                 padding='valid',
#                 activation='relu',
#                 strides=1))
#model.add(MaxPooling1D(pool_size=pool_size))
model.add(LSTM(lstm_output_size,dropout=0.2,recurrent_dropout=0.2))
model.add(Dense(lstm_output_size,activation="relu"))
model.add(Dense(1,activation="sigmoid"))

#model.add(Dense(1))
#model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# + colab={"base_uri": "https://localhost:8080/", "height": 182} colab_type="code" id="78SH0NQ5svGd" outputId="73001959-9bbd-4304-883c-bbf5ab4aa2ef"
print('Train RNN model...')
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
# -

# # Parte 2 - Challenge Kaggle

# +
import random
import glob
import itertools
import re
import gensim
import pickle
import abc

import scipy.sparse as sp

from collections import Counter
from sklearn.metrics import accuracy_score

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as init
import torch.nn.utils as utils
from torch.utils.data import DataLoader
from torch.utils.data import TensorDataset

torch.backends.cudnn.benchmark = True
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)

# +
main_path = './'

train_labels_path = main_path + 'train_labels.csv'
train_tree_path = main_path + 'train/'
train_source_path = main_path + 'train_source_tweets.txt'

test_tree_path = main_path + 'test/'
sample_labels_path = main_path + 'sample_submission.csv'
test_source_path = main_path + 'test_source_tweets.txt'
# -

include_errors = False


# # Create dataset

def remove_random(l,n):
    return random.sample(l,int(len(l)*(1-n)))


def truncate_list(l,to):
    if len(l)>=to:
        return remove_random(l,0.5)[:to+1]
    return l


def clean_line(text):
    
    text = text.replace("[", "")
    text = text.replace("]", "")
    text = text.replace("'", "")
    text = text.replace("\n", "")
    text = text.strip()
    
    return text


def read_data(path):
    
    f = lambda t: (1./(max(t, 0) + 1))
    
    data_child_array = []
    time_array = []
    
    root_tweet_id = ''
    
    for line in open(path, "r", encoding = "UTF-8"):
        
        line = clean_line(line)
        line_array = line.split("->")
        
        parent = line_array[0]
        parent_array = parent.split(",")
        parent_user_id = parent_array[0]
        parent_tweet_id = parent_array[1] 
        parent_time_delay = str(float(parent_array[2]))
        
        child = line_array[1]
        child_array = child.split(",")
        child_user_id = child_array[0]
        child_tweet_id = child_array[1] 
        child_time_delay = str(float(child_array[2]))
        
        
        if parent_user_id == "ROOT":
            
            source_claim = {"user_id" : child_user_id, "time_delay" : f(float(child_time_delay)), "tweet_id":child_tweet_id}
            root_user_id = child_user_id
        
        if parent_user_id !=root_user_id and parent_user_id != "ROOT":
            pkg = "%d:%.10f"%(int(parent_user_id),f(float(parent_time_delay)))
            data_child_array.append(pkg)
    
    #drop percent data here
    #data_child_array = remove_random(data_child_array,0.5)
    data_child_array = truncate_list(data_child_array,30)
    data_child_array.append(source_claim["user_id"]+":"+str(source_claim["time_delay"]))
    edges = ' '.join(data_child_array)
    tweet_id = int(source_claim['tweet_id'])
    
    line = "{}\t{}".format(tweet_id,edges)
    
    return line

trees_train = glob.glob(os.path.join(train_tree_path, "*.txt"))
trees_test = glob.glob(os.path.join(test_tree_path, "*.txt"))

# +
error_files_train = []
error_files_test = []

f = open("twitter_graph.txt", "w")

for file in trees_train:
    try:
        f.write(read_data(file)+"\n")
    except:
        error_files_train.append(file)
        print(file)

for file in trees_test:
    try:
        f.write(read_data(file)+"\n")
    except:
        error_files_test.append(file)
        print(file)

f.close()

error_files_train = [int(i.split('/')[-1][:-4]) for i in error_files_train]
error_files_test = [int(i.split('/')[-1][:-4]) for i in error_files_test]
# -

train_labels = pd.read_csv(train_labels_path)
train_source = pd.read_csv(train_source_path,sep='\t',header=None,names=['id','text'])
train_source['label'] = train_labels['label']
all_source_train = train_source.set_index('id').T.to_dict('list')

sample_labels = pd.read_csv(sample_labels_path)
test_source = pd.read_csv(test_source_path, sep='\t',header=None,names=['id','text'])
test_source['label'] = sample_labels['label']
all_source_test = test_source.set_index('id').T.to_dict('list')

if not include_errors:
    for _id in error_files_train:
        del all_source_train[_id]
    for _id in error_files_test:
        del all_source_test[_id]

# +
from sklearn.model_selection import train_test_split

keys = list(all_source_train.keys())
train_ids, dev_ids = train_test_split(keys, test_size=0.3, random_state=1)
# -

# # Preload Dataset

# +
w2v_dim = 300

dic = {
    'non-rumor': 0,
    'false': 1,
    'unverified': 2,
    'true': 3,
    'sample':4,
}


# -

def clean_str_cut(string, task=""):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)

    words = string.strip().lower().split()
    return words


def build_symmetric_adjacency_matrix(edges, shape):
    def normalize_adj(mx):
        """Row-normalize sparse matrix"""
        rowsum = np.array(mx.sum(1))
        r_inv_sqrt = np.power(rowsum, -0.5).flatten()
        r_inv_sqrt[np.isinf(r_inv_sqrt)] = 0.
        r_mat_inv_sqrt = sp.diags(r_inv_sqrt)
        return mx.dot(r_mat_inv_sqrt).transpose().dot(r_mat_inv_sqrt)

    adj = sp.coo_matrix((edges[:, 2], (edges[:, 0], edges[:, 1])), shape=shape, dtype=np.float32)
    # build symmetric adjacency matrix
    adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
    adj = normalize_adj(adj + sp.eye(adj.shape[0]))
    return adj.tocoo()


# +
X_tids = []
X_uids = []

X_train_tid, X_train_content, y_train = [], [], []
for _id in train_ids:
    text, label = all_source_train[_id]    
    X_tids.append(_id)
    X_train_tid.append(_id)
    X_train_content.append(clean_str_cut(text))
    y_train.append(dic[label])
    
    
X_dev_tid, X_dev_content, y_dev = [], [], []
for _id in dev_ids:
    text, label = all_source_train[_id]
    X_tids.append(_id)
    X_dev_tid.append(_id)
    X_dev_content.append(clean_str_cut(text))
    y_dev.append(dic[label]) 
    

X_test_tid, X_test_content, y_test = [], [], []
for _id in list(all_source_test.keys()):
    text, label = all_source_test[_id]
    X_tids.append(_id)
    X_test_tid.append(_id)
    X_test_content.append(clean_str_cut(text))
    y_test.append(dic[label])


with open("twitter_graph.txt", 'r', encoding='utf-8') as input:
    relation = []
    for line in input.readlines():
        tmp = line.strip().split()
        src = tmp[0]
        X_uids.append(src)

        for dst_ids_ws in tmp[1:]:
            dst, w = dst_ids_ws.split(":")
            X_uids.append(dst)
            relation.append([src, dst, w])    
    
X_id = list(set(X_tids + X_uids))
num_node = len(X_id)
print(num_node)
X_id_dic = {id:i for i, id in enumerate(X_id)}

relation = np.array([[X_id_dic[tup[0]], X_id_dic[tup[1]], tup[2]] for tup in relation])
relation = build_symmetric_adjacency_matrix(relation, shape=(num_node, num_node))

X_train_tid = np.array([X_id_dic[tid] for tid in X_train_tid])
X_dev_tid = np.array([X_id_dic[tid] for tid in X_dev_tid])
X_test_tid = np.array([X_id_dic[tid] for tid in X_test_tid])


# -

def vocab_to_word2vec(fname, vocab):
    """
    Load word2vec from Mikolov
    """
    word_vecs = {}
    model = gensim.models.KeyedVectors.load_word2vec_format(fname, binary=True)
    count_missing = 0
    for word in vocab:
        if model.__contains__(word):
            word_vecs[word] = model[word]
        else:
            #add unknown words by generating random word vectors
            count_missing += 1
            word_vecs[word] = np.random.uniform(-0.25, 0.25, w2v_dim)

    print(str(len(word_vecs) - count_missing)+" words found in word2vec.")
    print(str(count_missing)+" words not found, generated by random.")
    return word_vecs


def build_vocab_word2vec(sentences, w2v_path='numberbatch-en.txt'):
    """
    Builds a vocabulary mapping from word to index based on the sentences.
    Returns vocabulary mapping and inverse vocabulary mapping.
    """
    # Build vocabulary
    vocabulary_inv = []
    word_counts = Counter(itertools.chain(*sentences))
    # Mapping from index to word
    vocabulary_inv += [x[0] for x in word_counts.most_common() if x[1] >= 2]
    # Mapping from word to index
    vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}

    print("embedding_weights generation.......")
    word2vec = vocab_to_word2vec(w2v_path, vocabulary)     #
    embedding_weights = build_word_embedding_weights(word2vec, vocabulary_inv)
    return vocabulary, embedding_weights


def pad_sequence(X, max_len=50):
    X_pad = []
    for doc in X:
        if len(doc) >= max_len:
            doc = doc[:max_len]
        else:
            doc = [0] * (max_len - len(doc)) + doc
        X_pad.append(doc)
    return X_pad


def build_word_embedding_weights(word_vecs, vocabulary_inv):
    """
    Get the word embedding matrix, of size(vocabulary_size, word_vector_size)
    ith row is the embedding of ith word in vocabulary
    """
    vocab_size = len(vocabulary_inv)
    embedding_weights = np.zeros(shape=(vocab_size+1, w2v_dim), dtype='float32')
    #initialize the first row
    embedding_weights[0] = np.zeros(shape=(w2v_dim,) )

    for idx in range(1, vocab_size):
        embedding_weights[idx] = word_vecs[vocabulary_inv[idx]]
    print("Embedding matrix of size "+str(np.shape(embedding_weights)))
    return embedding_weights


def build_input_data(X, vocabulary):
    """
    Maps sentencs and labels to vectors based on a vocabulary.
    """
    x = [[vocabulary[word] for word in sentence if word in vocabulary] for sentence in X]
    x = pad_sequence(x)
    return x


def w2v_feature_extract(root_path, filename, w2v_path='twitter_w2v.bin'):
    """X_train_tid, X_train, y_train, \
    X_dev_tid, X_dev, y_dev, \
    X_test_tid, X_test, y_test, relation = read_corpus(root_path, filename)"""

    print("text word2vec generation.......")
    vocabulary, word_embeddings = build_vocab_word2vec(X_train + X_dev + X_test, w2v_path=w2v_path)
    pickle.dump(vocabulary, open(root_path + "/vocab.pkl", 'wb'))
    print("Vocabulary size: "+str(len(vocabulary)))

    print("build input data.......")
    X_train = build_input_data(X_train, vocabulary)
    X_dev = build_input_data(X_dev, vocabulary)
    X_test = build_input_data(X_test, vocabulary)

    pickle.dump([X_train_tid, X_train, y_train, word_embeddings, relation], open(root_path+"/train.pkl", 'wb') )
    pickle.dump([X_dev_tid, X_dev, y_dev], open(root_path+"/dev.pkl", 'wb') )
    pickle.dump([X_test_tid, X_test, y_test], open(root_path+"/test.pkl", 'wb') )


X_train_tid, X_train, y_train, \
X_dev_tid, X_dev, y_dev, \
X_test_tid, X_test, y_test, relation = X_train_tid, X_train_content, y_train, \
X_dev_tid, X_dev_content, y_dev, \
X_test_tid, X_test_content, y_test, \
relation

# +
print("text word2vec generation.......")
vocabulary, word_embeddings = build_vocab_word2vec(X_train + X_dev + X_test, w2v_path='twitter_w2v.bin')
pickle.dump(vocabulary, open("./vocab.pkl", 'wb'))
print("Vocabulary size: "+str(len(vocabulary)))

print("build input data.......")
X_train = build_input_data(X_train, vocabulary)
X_dev = build_input_data(X_dev, vocabulary)
X_test = build_input_data(X_test, vocabulary)

pickle.dump([X_train_tid, X_train, y_train, word_embeddings, relation], open("./train.pkl", 'wb') )
pickle.dump([X_dev_tid, X_dev, y_dev], open("./dev.pkl", 'wb') )
pickle.dump([X_test_tid, X_test, y_test], open("./test.pkl", 'wb') )


# -

# # Creando la red

# ## GAT

class SpecialSpmmFunction(torch.autograd.Function):
    """
    Special function for only sparse region backpropataion layer.
    """
    @staticmethod
    def forward(ctx, indices, values, shape, b):
        assert indices.requires_grad == False
        a = torch.sparse_coo_tensor(indices, values, shape)
        ctx.save_for_backward(a, b)
        ctx.N = shape[0]
        return torch.matmul(a, b)

    @staticmethod
    def backward(ctx, grad_output):
        a, b = ctx.saved_tensors
        grad_values = grad_b = None
        if ctx.needs_input_grad[1]:
            grad_a_dense = grad_output.matmul(b.t())
            edge_idx = a._indices()[0, :] * ctx.N + a._indices()[1, :]
            grad_values = grad_a_dense.view(-1)[edge_idx]

        if ctx.needs_input_grad[3]:
            grad_b = a.t().matmul(grad_output)
        return None, grad_values, None, grad_b



class SpecialSpmm(nn.Module):
    def forward(self, indices, values, shape, b):
        return SpecialSpmmFunction.apply(indices, values, shape, b)


class SpGraphAttentionLayer(nn.Module):
    """
    Sparse version GAT layer, similar to https://arxiv.org/abs/1710.10903
    """

    def __init__(self, in_features, out_features, dropout, alpha, concat=True):
        super(SpGraphAttentionLayer, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.alpha = alpha
        self.concat = concat

        self.W = nn.Parameter(torch.zeros(size=(in_features, out_features)))
        nn.init.xavier_normal_(self.W.data)

        self.a = nn.Parameter(torch.zeros(size=(1, 2*out_features)))
        nn.init.xavier_normal_(self.a.data)

        self.dropout = nn.Dropout(dropout)
        self.leakyrelu = nn.LeakyReLU(self.alpha)
        self.special_spmm = SpecialSpmm()


    def forward(self, input, adj):
        N = input.size()[0]
        edge = torch.LongTensor(adj.nonzero())

        h = torch.mm(input, self.W)
        # h: N x out
        assert not torch.isnan(h).any()

        # Self-attention on the nodes - Shared attention mechanism
        edge_h = torch.cat((h[edge[0, :], :], h[edge[1, :], :]), dim=1).t()
        # edge: 2*D x E

        edge_e = torch.exp(-self.leakyrelu(self.a.mm(edge_h).squeeze()))
        assert not torch.isnan(edge_e).any()
        # edge_e: E

        #e_rowsum = self.special_spmm(edge, edge_e, torch.Size([N, N]), torch.ones(size=(N, 1)).cuda())
        e_rowsum = self.special_spmm(edge, edge_e, torch.Size([N, N]), torch.ones(size=(N, 1)))
        # e_rowsum: N x 1

        edge_e = self.dropout(edge_e)
        # edge_e: E

        h_prime = self.special_spmm(edge, edge_e, torch.Size([N, N]), h)
        assert not torch.isnan(h_prime).any()
        # h_prime: N x out

        h_prime = h_prime.div(e_rowsum)
        # h_prime: N x out
        assert not torch.isnan(h_prime).any()

        if self.concat:
            # if this layer is not last layer,
            return F.elu(h_prime)
        else:
            # if this layer is last layer,
            return h_prime

    def __repr__(self):
        return self.__class__.__name__ + ' (' + str(self.in_features) + ' -> ' + str(self.out_features) + ')'


class GAT(nn.Module):

    def __init__(self, nfeat, uV, adj, hidden=16, nb_heads=8, n_output=300, dropout=0.2, alpha=0.3):
        """Sparse version of GAT."""
        super(GAT, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.uV = uV
        self.adj = adj
        self.user_tweet_embedding = nn.Embedding(self.uV, 300, padding_idx=0)
        init.xavier_uniform_(self.user_tweet_embedding.weight)

        self.attentions = nn.ModuleList([SpGraphAttentionLayer(in_features = nfeat,
                                                        out_features= hidden,
                                                        dropout=dropout,
                                                        alpha=alpha,
                                                        concat=True) for _ in range(nb_heads)])
        
        self.out_att = SpGraphAttentionLayer(hidden * nb_heads,
                                              n_output,
                                             dropout=dropout,
                                             alpha=alpha,
                                             concat=False)

    def forward(self, X_tid):
        #X = self.user_tweet_embedding(torch.arange(0, self.uV).long().cuda())
        X = self.user_tweet_embedding(torch.arange(0, self.uV).long())
        X = self.dropout(X)

        X = torch.cat([att(X, self.adj) for att in self.attentions], dim=1)
        X = self.dropout(X)

        X = F.elu(self.out_att(X, self.adj))
        X_ = X[X_tid]
        return X_


# ## TransformerBlock

class TransformerBlock(nn.Module):

    def __init__(self, input_size, d_k=16, d_v=16, n_heads=8, is_layer_norm=False, attn_dropout=0.1):
        super(TransformerBlock, self).__init__()
        self.n_heads = n_heads
        self.d_k = d_k if d_k is not None else input_size
        self.d_v = d_v if d_v is not None else input_size

        self.is_layer_norm = is_layer_norm
        if is_layer_norm:
            self.layer_morm = nn.LayerNorm(normalized_shape=input_size)

        self.W_q = nn.Parameter(torch.Tensor(input_size, n_heads * d_k))
        self.W_k = nn.Parameter(torch.Tensor(input_size, n_heads * d_k))
        self.W_v = nn.Parameter(torch.Tensor(input_size, n_heads * d_v))

        self.W_o = nn.Parameter(torch.Tensor(d_v*n_heads, input_size))
        self.linear1 = nn.Linear(input_size, input_size)
        self.linear2 = nn.Linear(input_size, input_size)

        self.dropout = nn.Dropout(attn_dropout)
        self.__init_weights__()
        print(self)

    def __init_weights__(self):
        init.xavier_normal_(self.W_q)
        init.xavier_normal_(self.W_k)
        init.xavier_normal_(self.W_v)
        init.xavier_normal_(self.W_o)

        init.xavier_normal_(self.linear1.weight)
        init.xavier_normal_(self.linear2.weight)

    def FFN(self, X):
        output = self.linear2(F.relu(self.linear1(X)))
        output = self.dropout(output)
        return output

    def scaled_dot_product_attention(self, Q, K, V, episilon=1e-6):
        '''
        :param Q: (*, max_q_words, n_heads, input_size)
        :param K: (*, max_k_words, n_heads, input_size)
        :param V: (*, max_v_words, n_heads, input_size)
        :param episilon:
        :return:
        '''
        temperature = self.d_k ** 0.5
        Q_K = torch.einsum("bqd,bkd->bqk", Q, K) / (temperature + episilon)
        Q_K_score = F.softmax(Q_K, dim=-1)  # (batch_size, max_q_words, max_k_words)
        Q_K_score = self.dropout(Q_K_score)

        V_att = Q_K_score.bmm(V)  # (*, max_q_words, input_size)
        return V_att


    def multi_head_attention(self, Q, K, V):
        bsz, q_len, _ = Q.size()
        bsz, k_len, _ = K.size()
        bsz, v_len, _ = V.size()

        Q_ = Q.matmul(self.W_q).view(bsz, q_len, self.n_heads, self.d_k)
        K_ = K.matmul(self.W_k).view(bsz, k_len, self.n_heads, self.d_k)
        V_ = V.matmul(self.W_v).view(bsz, v_len, self.n_heads, self.d_v)

        Q_ = Q_.permute(0, 2, 1, 3).contiguous().view(bsz*self.n_heads, q_len, self.d_k)
        K_ = K_.permute(0, 2, 1, 3).contiguous().view(bsz*self.n_heads, q_len, self.d_k)
        V_ = V_.permute(0, 2, 1, 3).contiguous().view(bsz*self.n_heads, q_len, self.d_v)

        V_att = self.scaled_dot_product_attention(Q_, K_, V_)
        V_att = V_att.view(bsz, self.n_heads, q_len, self.d_v)
        V_att = V_att.permute(0, 2, 1, 3).contiguous().view(bsz, q_len, self.n_heads*self.d_v)

        output = self.dropout(V_att.matmul(self.W_o)) # (batch_size, max_q_words, input_size)
        return output


    def forward(self, Q, K, V):
        '''
        :param Q: (batch_size, max_q_words, input_size)
        :param K: (batch_size, max_k_words, input_size)
        :param V: (batch_size, max_v_words, input_size)
        :return:  output: (batch_size, max_q_words, input_size)  same size as Q
        '''
        V_att = self.multi_head_attention(Q, K, V)

        if self.is_layer_norm:
            X = self.layer_morm(Q + V_att)  # (batch_size, max_r_words, embedding_dim)
            output = self.layer_morm(self.FFN(X) + X)
        else:
            X = Q + V_att
            output = self.FFN(X) + X
        return output


# # NeuralNetwork

class NeuralNetwork(nn.Module):

    def __init__(self):
        super(NeuralNetwork, self).__init__()
        self.best_acc = 0
        self.patience = 0
        self.init_clip_max_norm = None
        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    @abc.abstractmethod
    def forward(self):
        pass

    def fit(self, X_train_tid, X_train, y_train,
                  X_dev_tid, X_dev, y_dev):

        if torch.cuda.is_available():
            self.cuda()
        batch_size = self.config['batch_size']
        self.optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)

        X_train_tid = torch.LongTensor(X_train_tid)
        X_train = torch.LongTensor(X_train)
        y_train = torch.LongTensor(y_train)

        dataset = TensorDataset(X_train_tid, X_train, y_train)
        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

        loss_func = nn.CrossEntropyLoss()
        for epoch in range(self.config['epochs']):
            print("\nEpoch ", epoch+1, "/", self.config['epochs'])
            self.train()
            avg_loss = 0
            avg_acc = 0
            for i, data in enumerate(dataloader):
                with torch.no_grad():
                    #batch_x_tid, batch_x_text, batch_y = (item.cuda(device=self.device) for item in data)
                    batch_x_tid, batch_x_text, batch_y = (item.to(self.device) for item in data)

                self.optimizer.zero_grad()
                logit = self.forward(batch_x_tid, batch_x_text)
                loss = loss_func(logit, batch_y)
                loss.backward()
                self.optimizer.step()

                corrects = (torch.max(logit, 1)[1].view(batch_y.size()).data == batch_y.data).sum()
                accuracy = 100*corrects/len(batch_y)

                print('Batch[{}] - loss: {:.6f}  acc: {:.4f}%({}/{})'.format(i, loss.item(), accuracy, corrects, batch_y.size(0)))
                if i > 0 and i % 100 == 0:
                    self.evaluate(X_dev, y_dev)
                    self.train()

                avg_loss += loss.item()
                avg_acc += accuracy

                if self.init_clip_max_norm is not None:
                    utils.clip_grad_norm_(self.parameters(), max_norm=self.init_clip_max_norm)

            cnt = y_train.size(0) // batch_size + 1
            print("Average loss:{:.6f} average acc:{:.6f}%".format(avg_loss/cnt, avg_acc/cnt))
            if epoch >= 10 and self.patience > 3:
                print("Reload the best model...")
                self.load_state_dict(torch.load(self.config['save_path']))
                now_lr = self.adjust_learning_rate(self.optimizer)
                print(now_lr)
                self.patience = 0

            self.evaluate(X_dev_tid, X_dev, y_dev)


    def adjust_learning_rate(self, optimizer, decay_rate=.5):
        now_lr = 0
        for param_group in optimizer.param_groups:
            param_group['lr'] = param_group['lr'] * decay_rate
            now_lr = param_group['lr']
        return now_lr


    def evaluate(self, X_dev_tid, X_dev, y_dev):
        y_pred = self.predict(X_dev_tid, X_dev)
        acc = accuracy_score(y_dev, y_pred)

        if acc > self.best_acc:
            self.best_acc = acc
            self.patience = 0
            torch.save(self.state_dict(), self.config['save_path'])
            print(classification_report(y_dev, y_pred, target_names=self.config['target_names'], digits=5))
            print("Val set acc:", acc)
            print("Best val set acc:", self.best_acc)
            print("save model!!!")
        else:
            self.patience += 1


    def predict(self, X_test_tid, X_test):
        if torch.cuda.is_available():
            self.cuda()

        self.eval()
        y_pred = []
        #X_test_tid = torch.LongTensor(X_test_tid).cuda()
        X_test_tid = torch.LongTensor(X_test_tid)
        #X_test = torch.LongTensor(X_test).cuda()
        X_test = torch.LongTensor(X_test)

        dataset = TensorDataset(X_test_tid, X_test)
        dataloader = DataLoader(dataset, batch_size=50)

        for i, data in enumerate(dataloader):
            with torch.no_grad():
                #batch_x_tid, batch_x_text = (item.cuda(device=self.device) for item in data)
                batch_x_tid, batch_x_text = (item.to(self.device) for item in data)

            logits = self.forward(batch_x_tid, batch_x_text)
            predicted = torch.max(logits, dim=1)[1]
            y_pred += predicted.data.cpu().numpy().tolist()
        return y_pred


# # GLAN

class GLAN(NeuralNetwork):

    def __init__(self, config, adj):
        super(GLAN, self).__init__()
        self.config = config
        self.uV = adj.shape[0]
        embedding_weights = config['embedding_weights']
        V, D = embedding_weights.shape
        maxlen = config['maxlen']
        dropout_rate = config['dropout']

        self.mh_attention = TransformerBlock(input_size=300)
        self.word_embedding = nn.Embedding(V, D, padding_idx=0, _weight=torch.from_numpy(embedding_weights))

        self.relation_embedding = GAT(nfeat=300, uV=self.uV, adj=adj)

        self.convs = nn.ModuleList([nn.Conv1d(300, 100, kernel_size=K) for K in config['kernel_sizes']])
        self.max_poolings = nn.ModuleList([nn.MaxPool1d(kernel_size=maxlen - K + 1) for K in config['kernel_sizes']])

        self.dropout = nn.Dropout(dropout_rate)
        self.relu = nn.ReLU()

        self.fc1 = nn.Linear(600, 300)
        self.fc2 = nn.Linear(300, config['num_classes'])

        self.init_weight()
        print(self)

    def init_weight(self):
        init.xavier_normal_(self.fc1.weight)
        init.xavier_normal_(self.fc2.weight)


    def forward(self, X_tid, X_text):
        X_text = self.word_embedding(X_text) # (N*C, W, D)
        X_text = self.mh_attention(X_text, X_text, X_text)
        X_text = X_text.permute(0, 2, 1)

        rembedding = self.relation_embedding(X_tid)

        conv_block = [rembedding]
        for _, (Conv, max_pooling) in enumerate(zip(self.convs, self.max_poolings)):
            act = self.relu(Conv(X_text))
            pool = max_pooling(act)
            pool = torch.squeeze(pool)
            conv_block.append(pool)
        conv_feature = torch.cat(conv_block, dim=1)
        features = self.dropout(conv_feature)

        a1 = self.relu(self.fc1(features))
        d1 = self.dropout(a1)

        output = self.fc2(d1)
        return output

# # RUN

os.environ["CUDA_VISIBLE_DEVICES"] = ""

config = {
    'reg':0,
    'batch_size':64,
    'nb_filters':100,
    'kernel_sizes':[3, 4, 5],
    'dropout':0.3,
    'maxlen':50,
    'epochs':10,
    'num_classes':4,
    'target_names':['NR', 'FR', 'UR', 'TR']
}

# +
X_train_tid, X_train, y_train, word_embeddings, adj = X_train_tid, X_train, y_train, word_embeddings, relation

X_dev_tid, X_dev, y_dev = X_dev_tid, X_dev, y_dev

X_test_tid, X_test, y_test = X_test_tid, X_test, y_test

config['embedding_weights'] = word_embeddings

print("#nodes: ", adj.shape[0])
# -

# ## Run Model

# +
model_suffix = GLAN.__name__.lower().strip("text")
config['save_path'] = 'checkpoint/weights.best.twitter.'+ model_suffix

mod_nn = GLAN(config, adj)
# -

# !mkdir checkpoint

mod_nn.fit(X_train_tid, X_train, y_train,X_dev_tid, X_dev, y_dev)

# ## Make Predicction

mod_nn.load_state_dict(torch.load(config['save_path']))
y_pred = mod_nn.predict(X_test_tid, X_test)

# ## Save submission

dic = {
    0:'non-rumor',
    1:'false',
    2:'unverified',
    3:'true',
}
sample_pred_label = [dic[i] for i in y_pred]

# +
import pandas as pd
submit = pd.read_csv(sample_labels_path)

if not include_errors:
    for _id in error_files_test:
        indexNames = submit[ submit['id'] == _id ].index
        submit.drop(indexNames , inplace=True)

        sample_ids = submit['id']
        
    final_sub = pd.DataFrame ({'label':sample_pred_label+['unverified']*len(error_files_test),'id':list(sample_ids)+error_files_test }, columns = ['label','id'])
else:
    final_sub = pd.DataFrame ({'label':sample_pred_label,'id':sample_ids }, columns = ['label','id'])

# -

final_sub.to_csv('./submission_kaggle.csv', index=False, header=True)

# # Referencias
#
# * https://arxiv.org/abs/1909.04465
# * https://arxiv.org/abs/1706.03762
# * https://arxiv.org/abs/1710.10903

StarcoderdataJupyterScriptsDedupFiltered