import json
import gensim
import numpy as np
import string
import tsne as ts # local copy in the repo

%matplotlib inline

import matplotlib.pyplot as plt

data = []
with open("../data/yelp/yelp_academic_dataset_reviews.json") as handle:
    for line in handle.readlines():
        yelp = json.loads(line)
        data.append(yelp)

len(data)

data[0]

revs = [d[u'text'] for d in data]

revs[0]

"""
Explanation: Word2Vec Whirlwind Tour
Lynn Cherny (@arnicas, arnicas@gmail.com)
End of explanation
"""


## Don't do this is you already have the model file! Skip to the step after.

#model = gensim.models.Word2Vec(YelpReviews(data, 'text'), min_count=2, workers=2)

#model.save('yelp_w2v_model.mod')

#model.save_word2vec_format('yelp_w2vformat.mod')

# If you already have a model file, load it here:

model = gensim.models.Word2Vec.load_word2vec_format('../data/yelp/yelp_w2vformat.mod')

model.most_similar(positive=["greasy"], topn=20)

model.most_similar("waitress")

model.vocab.items()[0:5]

model.most_similar(['good', 'pizza'])

model.most_similar_cosmul(['good', 'pizza']) # less susceptible to extreme outliers

model.most_similar(['dog'])

model.most_similar(['salon'])

model.most_similar(positive=['man', 'queen'], negative=['woman'])

"""
Explanation: What is Word2Vec?
"Generally, word2vec is trained using something called a skip-gram model. The skip-gram model, pictures above, attempts to use the vector representation that it learns to predict the words that appear around a given word in the corpus. Essentially, it uses the context of the word as it is used in a variety of books and other literature to derive a meaningful set of numbers. If the “context” of two words is similar, they will have similar vector representations." (Source)
"In word2vec, a distributed representation of a word is used. Take a vector with several hundred dimensions (say 1000). Each word is representated by a distribution of weights across those elements. So instead of a one-to-one mapping between an element in the vector and a word, the representation of a word is spread across all of the elements in the vector, and each element in the vector contributes to the definition of many words.
If I label the dimensions in a hypothetical word vector (there are no such pre-assigned labels in the algorithm of course), it might look a bit like this:"
<img src="img/word2vec-distributed-representation.png">
Source
So that means we can do associative logic, or analogies, with these models:
<img src="img/word2vec-king-queen-vectors.png">
Specifically, a large enough model of the right kind of language (like a lot of news, or lots of books) will allow you to get "queen" from putting in man, king, woman...
Source
Creating a word2vec Model with Gensim
This takes a while.  Don't do it on the Yelp data, I already did it.  You can skip down to the place where we load the file!
End of explanation
"""


import nltk
from nltk.corpus import stopwords
english_stops = stopwords.words('english')

revs[0]

tokens = [nltk.word_tokenize(rev) for rev in revs]  # this takes a long time. don't run unless you're sure.

mystops = english_stops + [u"n't", u'...', u"'ve"]

def clean_tokens(tokens, stoplist):
    """ Lowercases, takes out punct and stopwords and short strings """
    return [token.lower() for token in tokens if (token not in string.punctuation) and 
                (token.lower() not in stoplist) and len(token) > 2]

clean = [clean_tokens(tok, mystops) for tok in tokens]

from nltk import Text

allclean = [y for x in clean for y in x] # flatten the list of lists
cleantext = Text(allclean)

mostcommon = cleantext.vocab().most_common()[0:1500]
mostcommon_words = [word[0] for word in mostcommon]

mostcommon_words[0:12]

# thing required to get the vectors for tsne

def get_vectors(words, model):
    # requires model be in the binary format, not gensim's
    word_vectors = []
    word_labels = []
    for word in words:
        if word in model:
            word_vectors.append( model[word] )
            word_labels.append(word)
    return word_vectors, word_labels

mymodel = gensim.models.Word2Vec.load_word2vec_format('../data/yelp/yelp_w2vformat.mod')
vectors, labels = get_vectors(mostcommon_words, mymodel)

# should be same as top words above
labels[:12]

res = ts.tsne(np.asfarray(vectors, dtype='float'), 2, 50, 20)

"""
Explanation: Now let's do some basic word sentiment stuff again... for the html side!
End of explanation
"""


from collections import defaultdict
sentiment = defaultdict(int)
with open('../data/sentiment_wordlists/AFINN-111.txt') as handle:
    for line in handle.readlines():
        word = line.split('\t')[0]
        polarity = line.split('\t')[1]
        sentiment[word] = int(polarity)

sentiment['pho']

sentiment['good']

sentiment['angry']

def render_json( vectors, labels, filename ):
    output = []
    vectors = np.array(vectors)
    for i in range(len(vectors)):
        new_hash = {}
        new_hash["word"] = str(labels[i])
        new_hash["x"] = int(vectors[i][0])
        new_hash["y"] = int(vectors[i][1])
        new_hash["sentiment"] = sentiment[str(labels[i])]
        output.append(new_hash)
    with open(filename, 'w') as handle:
        json.dump(output, handle)

render_json(res, labels, "../outputdata/yelp.json")

"""
Explanation: The "AFINN-111.txt" file is another sentiment file.
End of explanation
"""


plt.figure(figsize=(15, 15))
plt.scatter(res[:,0], res[:,1], s=10, color='gray', alpha=0.2)

"""
Explanation: TSNE Plotting Stuff
End of explanation
"""


