# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/talha1503/word2vec/blob/master/Word2Vec.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="-l9xwu03dVQP" colab_type="code" outputId="7f485b14-0d51-49e4-f1dd-d31ca5ca997b" colab={"base_uri": "https://localhost:8080/", "height": 34}
from google.colab import drive
drive.mount('/content/drive')

# + id="Pj0Xqix2dWmu" colab_type="code" colab={}
import numpy as np
import pandas as pd
import nltk
import os
import re
import operator

# + id="ylyQUIxpfYML" colab_type="code" outputId="f5d51264-7220-43e9-a6fd-34438887d60f" colab={"base_uri": "https://localhost:8080/", "height": 118}
nltk.download('corpus')
nltk.download('punkt')
nltk.download('stopwords')

# + id="36uIANlBdld4" colab_type="code" colab={}
base_path ='/content/drive/My Drive/Datasets/GOT_DATASET'

# + id="4jAHLG1MfHcB" colab_type="code" colab={}
corpus=""
counter = 0
for text_file in os.listdir(base_path):
  # if counter == 2:
  #   break
  with open(base_path+'/'+text_file,'r') as txt_file:
      corpus+=txt_file.read().replace('\n','')
  break
  # counter+=1

# + id="Dy1HY4bed45y" colab_type="code" colab={}
# corpus=""
# from bs4 import BeautifulSoup
# import requests

# base_url = "https://en.wikipedia.org/wiki/Deep_learning"
# r = requests.get(base_url)
# soup = BeautifulSoup(r.content,'html.parser')

# paragraphs = soup.find_all('p')
# for paragraph in paragraphs:
#     corpus+=paragraph.text


# + id="6wqN9KNBcnbI" colab_type="code" colab={}
# print(corpus)

# + id="FpoeSnn1ex_Q" colab_type="code" colab={}
from nltk.corpus import stopwords
def clean(text):
  text = text.lower()
  text = text.replace('“','')
  text = text.replace('”','')
  text = re.sub("hasn’t","has not",text)
  text = re.sub("can’t","can not",text)
  text = re.sub("wouldn’t","would not",text)
  text = re.sub("couldn’t","could not",text)
  text = re.sub("won’t","will not",text)
  text = re.sub("isn’t","is not",text)
  text = re.sub("i’ll","i will",text)
  text = re.sub("he’ll","he will",text)
  text = re.sub("she’ll","she will",text)
  text = re.sub("i’m","i am",text)
  text = re.sub("you’ll","you will",text)
  text = re.sub("hadn’t","had not",text)
  text = re.sub("don’t","do not",text)
  text = re.sub("here’s","here is",text)
  text = re.sub("where’s","where is",text)
  text = re.sub("that’s","that is",text)
  text = re.sub("it’s","it is",text)
  text = re.sub("he’s","he is",text)
  text = re.sub("she’s","she is",text)
  text = re.sub("what’s","what is",text)
  text = re.sub("i’ve","i have",text)
  text = re.sub("they’re","they are",text)
  text = re.sub("you’re","you are",text)
  text = re.sub("we’d","we would",text)
  text = re.sub("i’d","i would",text)
  text = re.sub(r'[^A-Za-z]',' ',text)
  text = text.split()
  text = [word for word in text if word not in set(stopwords.words('english'))]
  text = ' '.join(text)
  return text


# + id="w4NVAitjf0IL" colab_type="code" outputId="f99c7963-c4f4-442c-88ff-8a1eff0eb89a" colab={"base_uri": "https://localhost:8080/", "height": 34}
from nltk.tokenize import sent_tokenize,word_tokenize
tp_corpus = sent_tokenize(corpus)
tp_corpus = tp_corpus[:5000]
tp_corpus = ' '.join(tp_corpus)
cleaned_corpus = clean(tp_corpus)
# print(cleaned_corpus[:50])
print(len(sent_tokenize(tp_corpus)))
# print(len(tp_corpus))
# print(tp_corpus[:5])

# + id="LizVURx-f45G" colab_type="code" outputId="86b6eae3-7712-4ff4-be2a-cb6f9752c99e" colab={"base_uri": "https://localhost:8080/", "height": 34}
all_words = word_tokenize(cleaned_corpus)
print(len(all_words))

# + id="4JX9Jv9Og-aR" colab_type="code" colab={}
sentences  = sent_tokenize(tp_corpus)

cleaned_sentences = []
for sentence in sentences:
  cleaned_sentences.append(clean(sentence))  

# + id="q1hZAhpHicwW" colab_type="code" outputId="70fb94e8-53a2-456c-c244-c86b76f4123c" colab={"base_uri": "https://localhost:8080/", "height": 54}
print(cleaned_sentences[20:40])


# + id="JT1rlt-pmU1X" colab_type="code" outputId="20d4e9f4-083f-40ec-fa95-50526b7a5235" colab={"base_uri": "https://localhost:8080/", "height": 50}
all_words = list(set(all_words))
print("Total Vocabulary size is:",len(all_words))
print("Total number of sentences are:",len(cleaned_sentences))


# + id="WTkolPNaq9YN" colab_type="code" colab={}
def generate_data(window_size,all_words,cleaned_sentences):
  total_vocab = len(all_words)
  word2Ind = {word:i for i,word in enumerate(all_words)} 
  ind2Word = {i:word for i,word in enumerate(all_words)}
  x_train,y_train = [],[]
  for sent_index,sentence in enumerate(cleaned_sentences):
    for index,word in enumerate(sentence.split()):
      if word in set(all_words): 
        center_word_array = np.zeros(total_vocab)
        center_word_array[word2Ind[word]] = 1
        context_word_array = np.zeros(total_vocab)
        # print("center word: ",word)
        # print("Sentence is : ",sentence)
        for j in range(index-window_size,index+window_size+1):
          if j>=0 and j!=index and j<len(sentence.split()):
            if sentence.split()[j] in set(all_words):
              # print("Context word: ",sentence.split()[j])
              context_word_array[word2Ind[sentence.split()[j]]] = 1
        # data.append([center_word_array,context_word_array]) 
        x_train.append(center_word_array)
        y_train.append(context_word_array)

  return x_train,y_train,word2Ind,ind2Word


# + id="NUD2A2A-b4LH" colab_type="code" colab={}
x_train,y_train,word_to_index,index_to_word = generate_data(2,all_words,cleaned_sentences)

# + id="mKbOJ0mc-9co" colab_type="code" colab={}
# import pickle

# pickle.dump(x_train,open('x_train_word2vec.pkl','wb'))
# pickle.dump(y_train,open('y_train_word2vec.pkl','wb'))
# pickle.dump(word2Ind,open('word_to_index_word2vec.pkl','wb'))
# pickle.dump(ind2Word,open('index_to_word_word2vec.pkl','wb'))

# + id="oM6rGFPRUKfC" colab_type="code" outputId="dcb24d22-02e8-448f-e460-5265e9a99632" colab={"base_uri": "https://localhost:8080/", "height": 34}
np.seterr(divide='warn', invalid='warn',over='warn',under='warn')


# + id="1h9SOqJ043GZ" colab_type="code" colab={}
def cosine_similarity(vector1,vector2):
  return (np.dot(vector_1,vector_2)/(np.linalg.norm(vector_1)*np.linalg.norm(vector_2)))


# + id="8Q8AMO24qXzh" colab_type="code" colab={}
class Word2Vec():
  def __init__(self,dimensions,window_size,epochs,learning_rate,vocab_size,x_train,y_train,word2Ind,ind2Word):
    self.dimensions = dimensions
    self.window_size = window_size
    self.epochs = epochs
    self.learning_rate = learning_rate
    self.vocab_size = vocab_size
    self.x_train = x_train
    self.y_train = y_train
    self.word2Ind = word2Ind
    self.ind2Word = ind2Word

  def train(self):
    self.W1 = np.random.uniform(-1,1,(self.vocab_size,self.dimensions))
    self.W2 = np.random.uniform(-1,1,(self.dimensions,self.vocab_size))

    for i in range(self.epochs):
      self.loss = 0
      print("-"*80)
      for index,(x_train_sample,y_train_sample) in enumerate(zip(self.x_train,self.y_train)):
        y,u,h = self.forward(x_train_sample) 
        self.backward(y,h,x_train_sample,y_train_sample)
        context_words_included = 0
        for j in range(self.vocab_size):
          if y_train[index][j] == 1.0:
            self.loss+=(-1)*u[j]
            context_words_included+=1
        self.loss += context_words_included*np.log(np.sum(np.exp(u-u.max())))
        # self.loss += context_words_included*np.log(np.sum(u))
        # print("Loss is:",self.loss,end=" ")
        # print("Type is :",type(self.loss))
        # print("Training Example number--->",index)
      print("On Epoch:",i,end=" ")  
      print("Loss is:",self.loss)  

  def forward(self,x_input):print(corpus)
    h = np.dot(self.W1.T,x_input).reshape(self.dimensions,1)
    u = np.dot(self.W2.T,h).reshape(self.vocab_size,1)
    # u = np.log(np.max(u, 1e-9))

    # y = np.exp(u) / np.sum(np.exp(u), axis=0)print(corpus)
    y = self.exp_normalize(u)
    return y,u,h

  def exp_normalize(self,x):
    b = x.max()
    y = np.exp(x - b)
    return y / y.sum()


  def backward(self,y,h,x_train_sample,y_train_sample):    
    error = y - y_train_sample.reshape(self.vocab_size,1)
    dW2 = np.dot(h,error.T)
    dW1 = np.dot(x_train_sample.reshape(self.vocab_size,1),np.dot(self.W2,error).T)
    # dW1 = np.dot(self.W2,error)
    self.W1 = self.W1 - self.learning_rate*(dW1)
    self.W2 = self.W2 - self.learning_rate*(dW2)

  def return_vector(self,word):
    word_index = self.word2Ind[word]
    return self.W1[word_index]

  def most_similar(self,source_word,number_of_words=5):
    source_vector = self.return_vector(source_word)
    similarity_list = []
    for word,index in self.word2Ind.items():
      similarity_list.append((cosine_similarity(source_vector,self.return_vector(word)),word))
    similarity_list.sort(key = operator.itemgetter(0))
    required_words = similarity_list[:number_of_words+1]
    word_list = [word[1] for word in required_words]
    return word_list

# + id="3JFfP7oAx6ZB" colab_type="code" colab={}
w_vectors = Word2Vec(dimensions = 10,
                      window_size=2,
                      epochs=5,
                      learning_rate=0.01,
                      vocab_size=len(all_words),
                      x_train=x_train,
                      y_train=y_train,
                      ind2Word = index_to_word,
                      word2Ind = word_to_index
                     )

# + id="XEFzXpQSQRaB" colab_type="code" outputId="1cc733ef-0216-45b6-cda5-5b4b9795c039" colab={"base_uri": "https://localhost:8080/", "height": 185}
w_vectors.train()

# + id="w0nA6nQTn0uM" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 533} outputId="4ca3fa29-a270-4015-e738-6fc1b7957f34"
import matplotlib.pyplot as plt 
from sklearn.manifold import TSNE

width_in_inches = 8
height_in_inches = 8
dots_per_inch = 80

plt.figure(
          figsize=(width_in_inches, height_in_inches),
          dpi=dots_per_inch
          )

# num_words_to_plot = 20
# word_indices = np.random.randint(0,w_vectors.vocab_size,num_words_to_plot)
# for index in word_indices:
#   # word_embedding = w_vectors.return_vector(w_vectors.ind2Word[index]).reshape(-1,1)
#   # word_embedding = TSNE(n_components=2).fit_transform(word_embedding)
#   word_embedding = w_vectors.return_vector(w_vectors.ind2Word[index])
#   # print(word_embedding[0])
#   plt.scatter(word_embedding[5],word_embedding[6],marker='x',color='b')
#   plt.text(word_embedding[5],word_embedding[6],w_vectors.ind2Word[index],fontsize=9)

words_to_plot = ['stark','death','hanged','win','glory','lannister','war','throne','killed','bran','dragon','targaryen']
for word in words_to_plot:
  # word_embedding = w_vectors.return_vector(w_vectors.ind2Word[index]).reshape(-1,1)
  # word_embedding = TSNE(n_components=2).fit_transform(word_embedding)
  word_embedding = w_vectors.return_vector(word)
  plt.scatter(word_embedding[5],word_embedding[8],marker='x',color='b')
  plt.text(word_embedding[5],word_embedding[8],word,fontsize=9)


# + id="9lfO9qVnpe0Q" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="0d71661d-eba4-4fbe-d183-75565e1851d1"
plt.savefig('output1.png')

# + id="VI-euP_wvZfs" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered