!pip install nltk

"""
Explanation: 11. Semantics 1: words - Lab excercises
11.E1 Accessing WordNet using NLTK
11.E2 Using word embeddings
11.E3 Comparing WordNet and word embeddings
11.E1 Accessing WordNet using NLTK
<a id='11.E1'></a>
NLTK (Natural Language Toolkit) is a python library for accessing many NLP tools and resources. The NLTK WordNet interface is described here: http://www.nltk.org/howto/wordnet.html
The NLTK python package can be installed using pip:
End of explanation
"""


import nltk
nltk.download('wordnet')

"""
Explanation: Import nltk and use its internal download tool to get WordNet:
End of explanation
"""


from nltk.corpus import wordnet as wn

"""
Explanation: Import the wordnet module:
End of explanation
"""


club_synsets = wn.synsets('club')
print(club_synsets)

"""
Explanation: Access synsets of a word using the synsets function:
End of explanation
"""


for synset in club_synsets:
    print("{0}\t{1}".format(synset.name(), synset.definition()))

dog = wn.synsets('dog')[0]
dog.definition()

"""
Explanation: Each synset has a definition function:
End of explanation
"""


dog.lemmas()

"""
Explanation: List lemmas of a synset:
End of explanation
"""


dog.hypernyms()

dog.hyponyms()

"""
Explanation: List hypernyms and hyponyms of a synset
End of explanation
"""


list(dog.closure(lambda s: s.hypernyms()))

"""
Explanation: The closure method of synsets allows us to retrieve the transitive closure of the hypernym, hyponym, etc. relations:
End of explanation
"""


cat = wn.synsets('cat')[0]
dog.lowest_common_hypernyms(cat)

dog.common_hypernyms(cat)

dog.path_similarity(cat)

"""
Explanation: common_hypernyms and lowest_common_hypernyms work in relation to another synset:
End of explanation
"""


wn.all_synsets(pos='n')

for c, noun in enumerate(wn.all_synsets(pos='n')):
    if c > 5:
        break
    print(noun.name())

"""
Explanation: To iterate through all synsets, possibly by POS-tag, use all_synsets, which returns a generator:
End of explanation
"""


!wget http://sandbox.hlt.bme.hu/~recski/stuff/glove.6B.50d.txt.gz
!gunzip -f glove.6B.50d.txt.gz

"""
Explanation: Excercise (optional): use WordNet to implement the "Guess the category" game: the program lists lemmas that all share a hypernym, which the user has to guess.
11.E2 Using word embeddings
<a id='11.E2'></a>

Download and extract the word embedding glove.6B, which was trained on 6 billion words of English text using the GloVe algorithm.
End of explanation
"""


import numpy as np

def read_embedding(fn):
    words = []
    emb = []
    word_index = {}
    c = 0
    with open(fn, encoding='utf-8') as f:
        for line in f:
            fields = line.strip().split()
            emb.append(np.array([float(i) for i in fields[1:]], dtype='float32'))
            words.append(fields[0])
            word_index[fields[0]] = c
            c += 1

    print("read {0} lines".format(c))
    return np.array(words), word_index, np.array(emb)    

def normalize_embedding(emb):
    return emb / np.linalg.norm(emb, axis=1)[:,None]

words, word_index, emb = read_embedding('glove.6B.50d.txt')
emb = normalize_embedding(emb)

"""
Explanation: Read the embedding into a 2D numpy array. Word forms should be stored in a separate 1D array. Also create a word index, a dictionary that returns the index of each word in the embedding. Vectors should be normalized to a length of 1
End of explanation
"""


def vec_sim(w1, w2, word_index, emb):
    if w1 not in word_index or w2 not in word_index:
        return None
    return np.dot(emb[word_index[w1]], emb[word_index[w2]])

vec_sim('cat', 'dog', word_index, emb)

"""
Explanation: write a function that takes two words and the embedding as input and returns their cosine similarity
End of explanation
"""


def nearest_n(word, words, word_index, emb, n=5):
    try:
        w_index = word_index[word]
    except KeyError:
        return None
    w_vec = emb[w_index]

    distances = np.dot(emb, w_vec)
    indices = np.argsort(distances)[-n:][::-1]
    return [words[i] for i in indices]          

print(nearest_n('dog', words, word_index, emb))
print(nearest_n('king', words, word_index, emb))

"""
Explanation: Implement a function that takes a word as a parameter and returns the 5 words that are closest to it in the embedding space
End of explanation
"""


def embed_synset(synset, words, word_index, emb):
    word_set = [lemma.name() for lemma in synset.lemmas()]
    indices = filter(None, map(word_index.get, word_set))
    vecs = np.array([emb[i] for i in indices])
    if len(vecs) == 0:
        return None
    return np.mean(vecs, axis=0)

def embed_synsets(words, word_index, emb):
    return {synset: embed_synset(synset, words, word_index, emb) for synset in wn.all_synsets()}

synset_emb = embed_synsets(words, word_index, emb)

"""
Explanation: 11.E3 Vector similarity in WordNet
<a id='11.E3'></a>
Use the code written in 11.E2 to analyze word groups in WordNet:

Create an embedding of WordNet synsets by mapping each of them to the mean of their lemmas' vectors.
End of explanation
"""


def synset_sim(ss1, ss2, synset_emb):
    vec1 = synset_emb[ss1]
    vec2 = synset_emb[ss2]
    if vec1 is None or vec2 is None:
        return None
    return np.dot(vec1, vec2)

synset_sim(dog, cat, synset_emb)

"""
Explanation: write a function that measures the similarity of two synsets based on the cosine similarity of their vectors
End of explanation
"""


def nearest_n_synsets(synset, synset_emb, n=5):
    distances = [(synset_sim(synset, other, synset_emb), other) for other in wn.all_synsets() if synset != other]
    distances = [(sim, synset) for sim, synset in distances if not sim is None]
    return sorted(distances, reverse=True)[:n]

%%time
nearest_n_synsets(wn.synsets('penguin')[0], synset_emb, 10)

"""
Explanation: Write a function that takes a synset as input and retrieves the n most similar synsets, using the above embedding
End of explanation
"""


words_in_both = [word for word in wn.all_lemma_names() if word in word_index]

len(words_in_both)

import random
sample = random.sample(words_in_both, 100)

from scipy.stats import spearmanr

def compare_sims(sample, synset_emb, word_index, emb):
    vec_sims, ss_sims = [], []
    for w1 in sample:
        for w2 in sample:
            ss_sim = synset_sim(wn.synsets(w1)[0], wn.synsets(w2)[0], synset_emb)
            if ss_sim is None:
                continue
            v_sim = vec_sim(w1, w2, word_index, emb)
            vec_sims.append(v_sim)
            ss_sims.append(ss_sim)
    return spearmanr(vec_sims, ss_sims)

compare_sims(sample, synset_emb, word_index, emb)

"""
Explanation: Build the list of all words that are both in wordnet and the GloVe embedding. On a sample of 100 such words, measure Spearman correlation of synset similarity and vector similarity (use scipy.stats.spearmanr)
End of explanation
"""


