# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import os
import sys
import nltk
from nltk.corpus import stopwords

# GLOBALS
LOCAL_DATA_ROOT = '/Users/varunn/Documents/projects/pretrained_word_vectors/'
GLOVE_PATH = LOCAL_DATA_ROOT + 'glove.6B/'
GLOVE_INP_FN = GLOVE_PATH + 'glove.6B.50d.txt'
OUT_PATH = '/Users/varunn/Documents/NLP-data/'
VOCAB_FILE = OUT_PATH+'vocabulary_file_w2v.txt'

count = 1
for line in open(GLOVE_INP_FN):
    if count <= 5:
        print(line)
    else:
        break
    count += 1


# # Entity Extraction

class Embedding(object):
    def __init__(self,vocab_file,vectors_file, vocab_flag=True):
        if vocab_flag:
            words = []
            with open(vocab_file, 'r') as f:
                lines = [x.rstrip().split('\n') for x in f.readlines()]
                lines = [x[0] for x in lines]
                for line in lines:
                    current_words = line.split(' ')
                    words = list(set(words) | set(current_words))
            

        with open(vectors_file, 'r') as f:
            vectors = {}
            for line in f:
                vals = line.rstrip().split(' ')
                vectors[vals[0]] = [float(x) for x in vals[1:]]
        
        
        if not vocab_flag:
            words = vectors.keys()
        
        vocab_size = len(words)
        vocab = {w: idx for idx, w in enumerate(words)}
        ivocab = {idx: w for idx, w in enumerate(words)}

        vector_dim = len(vectors[ivocab[0]])
        W = np.zeros((vocab_size, vector_dim))
        for word, v in vectors.items():
            if (word == '<unk>') | (word not in vocab):
                continue
            W[vocab[word], :] = v
            

        # normalize each word vector to unit variance
        W_norm = np.zeros(W.shape)
        d = (np.sum(W ** 2, 1) ** (0.5))
        W_norm = (W.T / d).T
        
        if vocab_flag:
            for i in range(W.shape[0]):
                x = W[i, :]
                if sum(x) == 0:
                    W_norm[i, :] = W[i, :]

        self.W = W_norm
        self.vocab = vocab
        self.ivocab = ivocab


def find_similar_words(embed,text,refs):

    C = np.zeros((len(refs),embed.W.shape[1]))

    for idx, term in enumerate(refs):
        if term in embed.vocab:
            C[idx,:] = embed.W[embed.vocab[term], :]


    tokens = text.split(' ')
    scores = [0.] * len(tokens)

    for idx, term in enumerate(tokens):
        if term in embed.vocab:
            vec = embed.W[embed.vocab[term], :]
            cosines = np.dot(C,vec.T)
            score = np.mean(cosines)
            scores[idx] = score
    
    print(scores)

    return tokens[np.argmax(scores)]

examples = ["i am looking for a place in the north of town",
            "looking for indian restaurants",
            "Indian wants to go to an italian restaurant",
            "show me chinese restaurants",
            "show me chines restaurants in the north",
            "show me a mexican place in the centre",
            "i am looking for an indian spot called olaolaolaolaolaola",
            "search for restaurants",
            "anywhere in the west",
            "anywhere near 18328",
            "I am looking for asian fusion food",
            "I am looking a restaurant in 29432",
            "I am looking for mexican indian fusion",
            "central indian restaurant"]

examples = [x.lower() for x in examples]

fn = open(OUT_PATH+'vocabulary_file_w2v.txt', 'w')
for example in examples:
    fn.write(example)
    fn.write('\n')
fn.close()

embed = Embedding(VOCAB_FILE, GLOVE_INP_FN, False)

print(embed.W.shape)
print(len(embed.vocab))

test_example1 = 'looking for spanish restaurants'
test_example2 = 'looking for indian restaurants'
test_example3 = 'looking for south indian restaurants'
test_example4 = 'I want to find a chettinad restaurant'
test_example5 = 'chinese man looking for a indian restaurant'
refs = ["mexican","chinese","french","british","american"]
threshold = 0.2

# With stopwords
for example in [test_example1, test_example2, test_example3,
                test_example4, test_example5]:
    example = example.lower()
    print('text: ', example)
    print(find_similar_words(embed,example,refs))
    print('\n')

# With stopwords
stop = set(stopwords.words('english'))
for example in [test_example1, test_example2, test_example3,
                test_example4, test_example5]:
    print('text: ', example)
    example = " ".join([x.lower() for x in nltk.word_tokenize(example)
               if x not in stop])
    print(find_similar_words(embed,example,refs))
    print('\n')

find_similar_words(embed, 'fish food', refs)

# # Intent Detection

# +
import numpy as np

def sum_vecs(embed,text):

    tokens = text.split(' ')
    vec = np.zeros(embed.W.shape[1])

    for idx, term in enumerate(tokens):
        if term in embed.vocab:
            vec = vec + embed.W[embed.vocab[term], :]
    return vec


def get_centroid(embed,examples):

    C = np.zeros((len(examples),embed.W.shape[1]))
    for idx, text in enumerate(examples):
        C[idx,:] = sum_vecs(embed,text)

    centroid = np.mean(C,axis=0)
    assert centroid.shape[0] == embed.W.shape[1]
    return centroid


def get_intent(embed,text):
    intents = ['deny', 'inform', 'greet']
    vec = sum_vecs(embed,text)
    scores = np.array([ np.linalg.norm(vec-data[label]["centroid"]) for label in intents ])
    return intents[np.argmin(scores)]


# -

data={
  "greet": {
    "examples" : ["hello","hey there","howdy","hello","hi","hey","hey ho"],
    "centroid" : None
  },
  "inform": {
    "examples" : [
      "i'd like something asian",
      "maybe korean",
      "what mexican options do i have",
      "what italian options do i have",
      "i want korean food",
      "i want german food",
      "i want vegetarian food",
      "i would like chinese food",
      "i would like indian food",
      "what japanese options do i have",
      "korean please",
      "what about indian",
      "i want some vegan food",
      "maybe thai",
      "i'd like something vegetarian",
      "show me french restaurants",
      "show me a cool malaysian spot"
    ],
    "centroid" : None
  },
  "deny": {
    "examples" : [
      "nah",
      "any other places ?",
      "anything else",
      "no thanks"
      "not that one",
      "i do not like that place",
      "something else please",
      "no please show other options"
    ],
    "centroid" : None
  }
}
intents = ['greet', 'inform', 'deny']

examples = []
for intent in intents:
    examples = list(set(examples) | set(data[intent]['examples']))

examples

fn = open(VOCAB_FILE, 'w')
for example in examples:
    fn.write(example)
    fn.write('\n')
fn.close()

embed = Embedding(VOCAB_FILE, GLOVE_INP_FN, False)

for label in data.keys():
    data[label]["centroid"] = get_centroid(embed,data[label]["examples"])

data

for text in ["hey you","i am looking for chinese food","not for me"]:
    print("text : '{0}', predicted_label : '{1}'".format(text,get_intent(embed,text)))

text = "how do you do"
print("text : '{0}', predicted_label : '{1}'".format(text,get_intent(embed,text)))



StarcoderdataJupyterScriptsDedupFiltered