# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="7fjcTlyE3WvR"
# #Tokenizers
# Copyright 2021,2022 <NAME>, MIT License
#
# Reference 1 for word embedding:
# https://www.geeksforgeeks.org/python-word-embedding-using-word2vec/
#
# Reference 2 for cosine similarity:
# SciKit Learn cosine similarity documentation
#
# ***Upload text.txt before running the Notebook***

# + id="JKJ8Saf6vR9b"
#@title Pre-Requisistes
# !pip install gensim==3.8.3
import nltk
nltk.download('punkt')

# + id="7o7EeDUUu0Sh"
import math
import numpy as np
from nltk.tokenize import sent_tokenize, word_tokenize 
import gensim 
from gensim.models import Word2Vec 
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
import warnings 
warnings.filterwarnings(action = 'ignore') 

# + id="1NRomrXEJOxJ" colab={"base_uri": "https://localhost:8080/"} outputId="76407b05-1c34-4bfe-e143-a78f32eaa192"
#@title Word2Vec Tokenization
#‘text.txt’ file 
sample = open("text.txt", "r") 
s = sample.read() 

# processing escape characters 
f = s.replace("\n", " ") 

data = [] 
# sentence parsing
for i in sent_tokenize(f): 
	temp = [] 
	# tokenize the sentence into words 
	for j in word_tokenize(i): 
		temp.append(j.lower())
	data.append(temp)

# Creating Skip Gram model 
model2 = gensim.models.Word2Vec(data, min_count = 1, size = 512,window = 5, sg = 1) 
print(model2)


# + id="YcC_3JLcJTgw"
#@title Cosine Similarity
def similarity(word1,word2):
        cosine=False #default value
        try:
                a=model2[word1]
                cosine=True
        except KeyError:     #The KeyError exception is raised
                print(word1, ":[unk] key not found in dictionary")#False implied

        try:
                b=model2[word2]#a=True implied
        except KeyError:       #The KeyError exception is raised
                cosine=False   #both a and b must be true
                print(word2, ":[unk] key not found in dictionary")

        if(cosine==True):
                b=model2[word2]
                # compute cosine similarity
                dot = np.dot(a, b)
                norma = np.linalg.norm(a)
                normb = np.linalg.norm(b)
                cos = dot / (norma * normb)

                aa = a.reshape(1,512) 
                ba = b.reshape(1,512)
                #print("Word1",aa)
                #print("Word2",ba)
                cos_lib = cosine_similarity(aa, ba)
                #print(cos_lib,"word similarity")
          
        if(cosine==False):cos_lib=0;
        return cos_lib


# + id="fMfgbogHJVh-" colab={"base_uri": "https://localhost:8080/"} outputId="e63aca68-def1-4e5a-ab4f-c933e94c7e37"
#@title Case 0: Words in text and dictionary
word1="freedom";word2="liberty"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="4B7vvKxOLbYC" colab={"base_uri": "https://localhost:8080/"} outputId="a3371748-e482-42bb-ec6c-fba1da74a977"
#@title Word(s) Case 1: Word not in text or dictionary
word1="corporations";word2="rights"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="qkFIC79JCQJp" colab={"base_uri": "https://localhost:8080/"} outputId="a3eaf824-5d9e-4e3a-ec3c-8fbf8e8e1622"
#@title Case 2: Noisy Relationship 
word1="etext";word2="declaration"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="16uvrkFODNjv" colab={"base_uri": "https://localhost:8080/"} outputId="69df8db7-138a-43a4-d365-275e8ed18dcd"
#@title Case 3: word in text, not in dictionary
word1="pie";word2="logic"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="mKVPiEi-GZtf" colab={"base_uri": "https://localhost:8080/"} outputId="69ec4009-8ec4-4892-c8a4-5489a081f961"
#@title Case 4: Rare words
word1="justiciar";word2="judgement"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="0xZtAm3DHGJg" colab={"base_uri": "https://localhost:8080/"} outputId="7d70a9c7-3efa-4dab-a090-074a58b72149"
#@title Case 5: Replacing rare words
word1="judge";word2="judgement"
print("Similarity",similarity(word1,word2),word1,word2)

# + id="wOSID8kXHXWt" colab={"base_uri": "https://localhost:8080/"} outputId="8a8a5c75-58fb-4459-f4e7-c23fd6fee514"
#@title Case 6: Entailment
word1="pay";word2="debt"
print("Similarity",similarity(word1,word2),word1,word2)

StarcoderdataJupyterScriptsDedupFiltered