# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import time
from time import time
import os

import os
import gensim
from gensim.models import Word2Vec, KeyedVectors

from nltk.corpus import stopwords
from nltk import download
stop_words = stopwords.words('english')
from nltk import word_tokenize
download('punkt')  # Download data for tokenizer.

from gensim.similarities import WmdSimilarity

start_nb = time()
#https://markroxor.github.io/gensim/static/notebooks/WMD_tutorial.html

# +
# Pre-processing a document.

def preprocess(doc):
    doc = doc.lower()  # Lower the text.
    doc = word_tokenize(doc)  # Split into words.
    doc = [w for w in doc if not w in stop_words]  # Remove stopwords.
    doc = [w for w in doc if w.isalpha()]  # Remove numbers and punctuation.
    return doc


# -

relPath = 'data/1.0-billTitleSponsors.csv'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

#data_dir = '/Users/whs/Documents/GitHub/gpo-ai.com/data/processed/1.0-billTitleSponsors.csv'
#data_dir = 'C:\\Users\\wsolomon\\Desktop\\DataScience\\projects\\loc_challenge-master\\1.0-billTitleSponsors.csv'
relPath = 'data/1.0-billTitleSponsors.csv'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)
df = pd.read_csv(absPath)
df.head()

# +
start = time()

relPath = 'model/GoogleNews-vectors-negative300.bin.gz'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

#path = '/Users/whs/Documents/GitHub/gpo-ai.com/data/processed/GoogleNews-vectors-negative300.bin.gz'
if not os.path.exists(absPath):
    raise ValueError("SKIP: You need to download the google news model")
    
model = gensim.models.KeyedVectors.load_word2vec_format(absPath, binary=True)

print('Cell took %.2f seconds to run.' % (time() - start))
# -

doc1 = df['texts'][3]
doc2 = df['texts'][4]
distance = model.wmdistance(doc1,doc2)
print('distance = %.4f' % distance)

print(df.shape)
df = df.dropna()
df.shape

#preprocess(df.texts.values[1000])
df.texts.values[1000]

# +
start = time()

wmd_corpus = [preprocess(doc) for doc in df.texts.values]
print('Docs in Corpus: ',len(wmd_corpus))
num_best = 4
instance = WmdSimilarity(wmd_corpus, model, num_best=num_best)

print('Cell took %.2f seconds to run.' % (time() - start))

# +
start = time()

sent = 'guns'
query = preprocess(sent)

sims = instance[query]  # A query is simply a "look-up" in the similarity class.
print(sims)
#print('Cell took %.2f seconds to run.' % (time() - start))

# Print the query and the retrieved documents, together with their similarities.
print('Query:')
print(sent)
for i in range(num_best):
    print()
    print('sim = %.4f' % sims[i][1])
    print(df.title.values[sims[i][0]])
    print(df.sponsers.values[sims[i][0]])

# +
relPath = 'model/1.0-whs-W2VModel'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

model.save(absPath)

# +
relPath = 'model/1.1-whs-Sim_Instance'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

instance.save(absPath)
# -



# +
w2v_relPath = 'model/GoogleNews-vectors-negative300.bin.gz'
w2v_absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

relPath = 'model/1.0-whs-W2VModel'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)
#modelX = KeyedVectors.load_word2vec_format(w2v_absPath, binary=True) #.load(absPath)

relPath = 'model/1.1-whs-Sim_Instance'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)
instanceX = WmdSimilarity.load(absPath)

relPath = 'data/1.0-billTitleSponsors.csv'
absPath = os.path.join(os.path.dirname(os.getcwd()), relPath)

df = pd.read_csv(absPath)

# +
start = time()

sent = 'I like sports'
query = preprocess(sent)

sims = instanceX[query]  # A query is simply a "look-up" in the similarity class.
print(sims)
#print('Cell took %.2f seconds to run.' % (time() - start))

# Print the query and the retrieved documents, together with their similarities.
print('Query:')
print(sent)
for i in range(num_best):
    print()
    print('sim = %.4f' % sims[i][1])
    print(df.title.values[sims[i][0]])
    print(df.sponsers.values[sims[i][0]])


print('Cell took %.2f seconds to run.' % (time() - start))

# +
start = time()

modelX = Word2Vec.load(path)
from gensim.similarities import WmdSimilarity
wmd_corpus = [preprocess(doc) for doc in df.title.values]
print(len(wmd_corpus))
num_best = 10
instanceX = WmdSimilarity(wmd_corpus, modelX, num_best=10)


print('Cell took %.2f seconds to run.' % (time() - start))

# +
start = time()

#model = Word2Vec.load(fname)
path = '/Users/whs/Documents/GitHub/gpo-ai.com/models/1.1-whs-Sim_Instance'
instanceX = WmdSimilarity.load(path)
sent = 'guns'
query = preprocess(sent)

sims = instanceX[query]  # A query is simply a "look-up" in the similarity class.
print(sims)
#print('Cell took %.2f seconds to run.' % (time() - start))

# Print the query and the retrieved documents, together with their similarities.
print('Query:')
print(sent)
for i in range(num_best):
    print()
    print('sim = %.4f' % sims[i][1])
    print(df.title.values[sims[i][0]])
    print(df.sponsers.values[sims[i][0]])
# -



StarcoderdataJupyterScriptsDedupFiltered