# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Named Entity Recognition using RNN

# ## Import Required Libraries

# + id="0YlBAoRG7P_D"
import warnings
warnings.filterwarnings("ignore")

import numpy as np

from matplotlib import pyplot as plt

from nltk.corpus import brown
from nltk.corpus import treebank
import nltk
import seaborn as sns

from gensim.models import KeyedVectors

from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Embedding
from keras.layers import Dense, Input
from keras.layers import TimeDistributed
from keras.layers import LSTM, GRU, Bidirectional, SimpleRNN, RNN
from keras.models import Model
from keras.preprocessing.text import Tokenizer

from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
# -

# ## Dataset

# + id="Szoops5X7hf3"
file1 = open("eng.train", "r")
X = [] # contains the words
Y = [] # contains corresponding tags
x = []
y = []
for i in file1:
  s = i.split(" ")
  if s[0]!='.' and len(s)==4:
    x.append(s[0])
    y.append(s[3][:-1])
  else:
    X.append(x)
    Y.append(y)
    x = []
    y = []
#print(X)
#print(Y)

# + id="_9AmKZmN-M9n"
num_words = len(set([word.lower() for sentence in X for word in sentence]))
num_tags   = len(set([word.lower() for sentence in Y for word in sentence]))

# + colab={"base_uri": "https://localhost:8080/"} id="vYce51Oa-Ree" outputId="99a60321-cf6b-4a0f-d588-75ebac337d02"
print("Total number of tagged sentences: {}".format(len(X)))
print("Vocabulary size: {}".format(num_words))
print("Total number of tags: {}".format(num_tags))

# -

# ## Encoding

# + id="7NI7ngWF-gj5"
# encode X

word_tokenizer = Tokenizer()                      # instantiate tokeniser
word_tokenizer.fit_on_texts(X)                    # fit tokeniser on data
X_encoded = word_tokenizer.texts_to_sequences(X)  # use the tokeniser to encode input sequence

# + id="uw81s_xm-jeF"
# encode Y

tag_tokenizer = Tokenizer()
tag_tokenizer.fit_on_texts(Y)
Y_encoded = tag_tokenizer.texts_to_sequences(Y)

# + colab={"base_uri": "https://localhost:8080/"} id="uYgF-C1S--41" outputId="6173dbd1-73cf-450d-ee20-99cae49a4152"
# make sure that each sequence of input and output is same length

different_length = [1 if len(input) != len(output) else 0 for input, output in zip(X_encoded, Y_encoded)]
print("{} sentences have disparate input-output lengths.".format(sum(different_length)))

# + colab={"base_uri": "https://localhost:8080/"} id="vlinzR4D_D9O" outputId="4273b4c2-af6b-40ed-923a-5717515636ef"
# check length of longest sentence
lengths = [len(seq) for seq in X_encoded]
print("Length of longest sentence: {}".format(max(lengths)))
# -

# ## Padding

# + id="HvvJjjAC_MR9"
MAX_SEQ_LENGTH = 100  # sequences greater than 100 in length will be truncated

X_padded = pad_sequences(X_encoded, maxlen=MAX_SEQ_LENGTH, padding="pre", truncating="post")
Y_padded = pad_sequences(Y_encoded, maxlen=MAX_SEQ_LENGTH, padding="pre", truncating="post")


# + id="bg3WVdKT_ThG"
# assign padded sequences to X and Y
X, Y = X_padded, Y_padded
# -

# ## Word2Vec Model

# + id="2JZxBC8sCKlP"
# word2vec

path = './GoogleNews-vectors-negative300.bin'


# load word2vec using the following function present in the gensim library
word2vec = KeyedVectors.load_word2vec_format(path, binary=True)


# + id="s-ohn5fPDWxS"
#assign word vectors from word2vec model

EMBEDDING_SIZE  = 300  # each word in word2vec model is represented using a 300 dimensional vector
VOCABULARY_SIZE = len(word_tokenizer.word_index) + 1

# create an empty embedding matix
embedding_weights = np.zeros((VOCABULARY_SIZE, EMBEDDING_SIZE))

# create a word to index dictionary mapping
word2id = word_tokenizer.word_index

# # copy vectors from word2vec model to the words present in corpus
for word, index in word2id.items():
    try:
        embedding_weights[index, :] = word2vec[word]
    except KeyError:
        pass

# + colab={"base_uri": "https://localhost:8080/"} id="2RY5cYD9DbEb" outputId="af507f97-4e1a-4c11-d740-50eb98a4eeab"
# check embedding dimension
print("Embeddings shape: {}".format(embedding_weights.shape))

# + colab={"base_uri": "https://localhost:8080/"} id="-BdwOh4_DhvR" outputId="366f49a8-bc05-4f5e-fc14-feee324a7947"
# let's look at an embedding of a word
embedding_weights[word_tokenizer.word_index['economy']]


# + id="Bul6DtKIDqlT"
# use Keras' to_categorical function to one-hot encode Y
Y = to_categorical(Y)

# + colab={"base_uri": "https://localhost:8080/"} id="rAe-D14iDzDr" outputId="a9f17ab4-602e-4291-f630-5344fc6d744a"
# print Y of the first output sequqnce
print(Y.shape)
# -

# ## Splitting Dataset

# + id="7PeXpHpgD246"
# split entire data into training and testing sets
TEST_SIZE = 0.15
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=TEST_SIZE, random_state=4)

# + id="RMRHsYnIEIwm"
# split training data into training and validation sets
VALID_SIZE = 0.15
X_train, X_validation, Y_train, Y_validation = train_test_split(X_train, Y_train, test_size=VALID_SIZE, random_state=4)


# + id="mjlyGel6EW-i"
# total number of tags
NUM_CLASSES = Y.shape[2]
# -

# ## RNN Model

# + id="xprqa4WWE5kp"
# create architecture

rnn_model = Sequential()

# create embedding layer - usually the first layer in text problems
rnn_model.add(Embedding(input_dim     =  VOCABULARY_SIZE,         # vocabulary size - number of unique words in data
                        output_dim    =  EMBEDDING_SIZE,          # length of vector with which each word is represented
                        input_length  =  MAX_SEQ_LENGTH,          # length of input sequence
                        trainable     =  True                     # True - update the embeddings while training
))

# add an RNN layer which contains 64 RNN cells
rnn_model.add(SimpleRNN(64, 
              return_sequences=True  # True - return whole sequence; False - return single output of the end of the sequence
))

# add time distributed (output at each sequence) layer
rnn_model.add(TimeDistributed(Dense(NUM_CLASSES, activation='softmax')))
# -

# ## Compiling the Model

# + id="yvQUQpB5E_aN"
rnn_model.compile(loss      =  'categorical_crossentropy',
                  optimizer =  'adam',
                  metrics   =  ['acc'])
# -

# ## Summary of the Model

# + colab={"base_uri": "https://localhost:8080/"} id="dbCuUnkhFK8s" outputId="6203c25e-081a-4a83-e5a0-20c799945d3b"
# check summary of the model
rnn_model.summary()
# -

# ## Training the Model

# + colab={"base_uri": "https://localhost:8080/"} id="kw3F89ajFRx7" outputId="625a0b20-df09-47c4-f392-9d2219262dc9"
rnn_training = rnn_model.fit(X_train, Y_train, batch_size=128, epochs=10, validation_data=(X_validation, Y_validation))
# -

# ## Plot

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="f33uSZkqFYYT" outputId="17ff2529-c0f0-42af-db35-28a99a247c66"
# visualise training history
plt.plot(rnn_training.history['acc'])
plt.plot(rnn_training.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc="lower right")
plt.show()

# -

# ## Saving the Model

rnn_model.save('ner-rnn.h5')

StarcoderdataJupyterScriptsDedupFiltered