# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="bIAYbYajk1w9" colab_type="text"
# <div class="alert alert-block alert-info" style="margin-top: 20px">
#
#       
# | Name | Description | Date
# | :- |-------------: | :-:
# |<NAME>| Convolutional Neural Networks - 6th  | On 23rd of August 2019 | width="750" align="center"></a></p>
# </div>
#
# # Convolutional Neural Networks
# - IMDB review sentiment classification with CNN
#   - Up to last time, we have tried to classify images in the CIFAR-10 dataset with CNN. However, CNNs are not only useful in classifying and recognizing images, but also processing data with temporal dependencies, such as text data.
#   - Here, let's try classifying movie review data with CNN

# + id="GVU5-yp3N89I" colab_type="code" outputId="56619a07-b42c-438e-d355-663ab457678b" executionInfo={"status": "ok", "timestamp": 1566585075665, "user_tz": 240, "elapsed": 2126, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBrdPXZmzhldkZcu5l9nrnO7t-Ls96No7O8kRuZ=s64", "userId": "14585138350013583795"}} colab={"base_uri": "https://localhost:8080/", "height": 119}
# !pip3 install torch torchvision

# + id="8yy37hEYOEiQ" colab_type="code" outputId="d695e86a-a98a-4951-d9cf-185ef8130dd9" executionInfo={"status": "ok", "timestamp": 1545933208234, "user_tz": 420, "elapsed": 1304, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "18268696804115368229"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
import numpy as np
import pandas as pd
import torch, torchvision
import torch.nn as nn
import torch.nn.functional as F
torch.__version__

# + [markdown] id="ewrw93tt2BfV" colab_type="text"
# ## 1. Import & process dataset
# - IMDB review dataset for sentiment analysis
#   - [source](http://ai.stanford.edu/~amaas/data/sentiment/)
#   - Let's cheat a while and use dataset provided by Keras

# + id="SxDHXFEsf5em" colab_type="code" outputId="6995c6a3-d576-4267-840a-f5362e03ef28" executionInfo={"status": "ok", "timestamp": 1545933216936, "user_tz": 420, "elapsed": 8829, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "18268696804115368229"}} colab={"base_uri": "https://localhost:8080/", "height": 52}
from keras.datasets import imdb
from keras.preprocessing import sequence

num_words = 10000
maxlen = 50

(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words = num_words)

X_train = sequence.pad_sequences(X_train, maxlen = maxlen, padding = 'pre')
X_test = sequence.pad_sequences(X_test, maxlen = maxlen, padding = 'pre')
    
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)


# + [markdown] id="9nznVMdo5edZ" colab_type="text"
# ## 2. Creating CNN model and training
#
# - Create and train CNN model for sentence classification, with one convolutional & average pooling layer
# - Model architecture is adopted from [Kim 2015](https://www.aclweb.org/anthology/D14-1181)
#
# ![](https://3qeqpr26caki16dnhd19sv6by6v-wpengine.netdna-ssl.com/wp-content/uploads/2017/08/Example-of-a-CNN-Filter-and-Polling-Architecture-for-Natural-Language-Processing.png)

# + id="tdmsy2B3weeK" colab_type="code" colab={}
class imdbTrainDataset(torch.utils.data.Dataset):
  def __init__(self):
    self.X = X_train
    self.y = y_train
  
  def __getitem__(self, idx):
    return self.X[idx], self.y[idx]
  
  def __len__(self):
    return len(self.X)
  
class imdbTestDataset(torch.utils.data.Dataset):
  def __init__(self):
    self.X = X_test
    self.y = y_test
  
  def __getitem__(self, idx):
    return self.X[idx], self.y[idx]
  
  def __len__(self):
    return len(self.X)


# + id="xFBmmDxKw1t2" colab_type="code" colab={}
# create dataset & dataloader instances
train_dataset = imdbTrainDataset()
test_dataset = imdbTestDataset()

train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = 128, shuffle = True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = 128, shuffle = False)


# + id="AQawpMRPI7jm" colab_type="code" colab={}
# create CNN with one convolution/pooling layer
class net(nn.Module):
  def __init__(self, input_dim, num_words, embedding_dim, num_filters, kernel_size, stride):
    super(net, self).__init__()
    self.input_dim = input_dim
    self.embedding_dim = embedding_dim
    
    conv_output_size = int((input_dim - kernel_size)/stride) + 1   # first conv layer output size
        
    self.embedding = nn.Embedding(num_words, self.embedding_dim)
    self.conv = nn.Conv2d(1, num_filters, kernel_size = (kernel_size, self.embedding_dim), stride = stride)     
    self.pool = nn.MaxPool2d((conv_output_size, 1))                # Max-over-time pooling (FYI: avg pooling also works)
    self.relu = nn.ReLU()
    self.dense = nn.Linear(num_filters, 2)     
    
  def forward(self, x):
    x = self.embedding(x)                                   # project to word embedding space
    x = x.view(-1, 1, self.input_dim, self.embedding_dim)   # resize to fit into convolutional layer
    x = self.conv(x)
    x = self.relu(x)
    x = self.pool(x) 
    x = x.view(x.size(0), -1)   # resize to fit into final dense layer
    x = self.dense(x)
    return x


# + id="rP0Gt5E9ajmd" colab_type="code" colab={}
# hyperparameters
DEVICE = torch.device('cuda')
INPUT_DIM = maxlen
NUM_FILTERS = 64
KERNEL_SIZE = 3
STRIDE = 1
EMBEDDING_DIM = 50
NUM_WORDS = num_words
LEARNING_RATE = 1e-3
NUM_EPOCHS = 30              

# + id="cPBm8qDrSWsi" colab_type="code" colab={}
model = net(INPUT_DIM, NUM_WORDS, EMBEDDING_DIM, NUM_FILTERS, KERNEL_SIZE, STRIDE).to(DEVICE)
criterion = nn.CrossEntropyLoss()   # do not need softmax layer when using CEloss criterion
optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)

# + id="SEBtAPYCFeic" colab_type="code" outputId="d94310e4-194c-455d-9830-a13d3c638039" executionInfo={"status": "ok", "timestamp": 1545933874196, "user_tz": 420, "elapsed": 210646, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "18268696804115368229"}} colab={"base_uri": "https://localhost:8080/", "height": 545}
# training for NUM_EPOCHS
for i in range(NUM_EPOCHS):
  temp_loss = []
  for (x, y) in train_loader:
    x, y = x.long().to(DEVICE), y.to(DEVICE)  # beware that input to embedding should be type 'long'
    outputs = model(x)
    loss = criterion(outputs, y)
    temp_loss.append(loss.item())
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
  print("Loss at {}th epoch: {}".format(i, np.mean(temp_loss)))

# + [markdown] id="qpAJUiHm529m" colab_type="text"
# ## 3. Evaluation
# - Evaluate the trained CNN model with accuracy score 
#   - Store probability of each instance to a list and compare it with true y label

# + id="txXH3dknFpSx" colab_type="code" outputId="35322a5b-a28c-463f-85c9-7b01b6cb66d8" executionInfo={"status": "ok", "timestamp": 1545933445208, "user_tz": 420, "elapsed": 964, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "18268696804115368229"}} colab={"base_uri": "https://localhost:8080/", "height": 72}
y_pred, y_true = [], []
with torch.no_grad():
  for x, y in test_loader:
    x, y = x.long().to(DEVICE), y.to(DEVICE)       # beware that input to embedding should be type 'long'
    outputs = F.softmax(model(x)).max(1)[-1]       # predicted label
    y_true += list(y.cpu().numpy())                # true label
    y_pred += list(outputs.cpu().numpy())   

# + id="HV1s3xf5Frkl" colab_type="code" outputId="90d07e53-a813-40e9-8227-09cbccd2fa7a" executionInfo={"status": "ok", "timestamp": 1545933447202, "user_tz": 420, "elapsed": 986, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "18268696804115368229"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# evaluation result
from sklearn.metrics import accuracy_score
accuracy_score(y_true, y_pred)

StarcoderdataJupyterScriptsDedupFiltered