import spacy
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sys
import codecs
import math

from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.manifold import TSNE
from gensim.models import KeyedVectors

# Display plots in this notebook, instead of externally. 
from pylab import rcParams
rcParams['figure.figsize'] = 16, 8
%matplotlib inline
nlp = spacy.load('es')

"""
Explanation: Introduction to Spanish NLP Workshop
First let's load spacy and a Spanish parsing model, this may take a while
End of explanation
"""


doc = nlp('Las naranjas y las manzanas se parecen')
pd.DataFrame([[word.text, word.tag_, word.pos_] for word in doc], columns=['Token', 'TAG', 'POS'])

"""
Explanation: Tokenization and POS Tagging
Now we can a pick a sentence and analyze it. A sentence is split into tokens (units of meaning: could be more than one word in some cases) and we can get the syntactic function of each one. This is called POS (Part of Speech) tagging
End of explanation
"""


pd.DataFrame([[word.text, math.exp(word.prob), word.prob] for word in doc], columns=['Token', 'Prob', 'Log Prob'])

"""
Explanation: Language Models
Anothe key concept in NLP is that of Language Models. It is a function that is able to tell us the likelihood of a sentence of appearing in the real word. One such model, albeit a very simple one, is the one that multiplies the probabilities of every token in the sentence
End of explanation
"""


doc2 = nlp(u"La premier alemana Angela Merkel visitó Buenos Aires esta semana")
for ent in doc2.ents:
    print('{} \t {}'.format(ent, ent.label_))

"""
Explanation: Names Entity Recognition
We are also interested in extracting the proper entities that appear in a phrase and this problem is referred as Named Entity Recognition or NER. Here's and example with different kinds of entities such as people and locations
End of explanation
"""


orange = doc[1]
apple = doc[4]
orange.similarity(apple)

"""
Explanation: Word Embeddings
Embeddings are the silver bullet brought from the world of deep learning into the realm of NLP through the word2vec algorithm. They are a meaningful
representation of tokens as a dense high dimensional vector. They can be obtained from large unlabelled corpora easily available
on the web and are able to express the sintactic, semantic, morpholical and even polysemic richness of a word in a way that 
a computer can understand
End of explanation
"""


doc.similarity(doc2)

"""
Explanation: One amazing property of word vectors is that they represent analogies really well, for instance:

Argentina - Macri = Alemania - Merkel
Reina - Mujer = Rey - Hombre

We can also average out the vectors in a whole sentence to get a similarity between them. NLP methods that disregard the word order such as this one are commonly referred as Bag of Words
End of explanation
"""


from bs4 import BeautifulSoup
import urllib.request
html = urllib.request.urlopen('http://api-editoriales.clarin.com/api/instant_articles?limit=100').read()
soup = BeautifulSoup(html, 'html.parser')
news = [
    BeautifulSoup(c.text, 'html.parser').text.split('function', 1)[0] 
    for item in soup.findAll('item') 
        for c in item.children if c.name == 'content:encoded'
]
news[0][0:100]

corpus = nlp('\n'.join(news))

visited = {}
nouns = []
for word in corpus:
    if word.pos_.startswith('N') and len(word.string) < 15:
        token = word.string.strip().lower()
        if token in visited:
            visited[token] += 1
            continue
        else:
            visited[token] = 1
            nouns.append(word)
nouns = sorted(nouns, key=lambda w: -visited[w.string.strip().lower()])[:150]
pd.DataFrame([[w.text, visited[w.string.strip().lower()]] for w in nouns], columns=['Noun', 'Freq'])

def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
    assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
    plt.figure(figsize=(18, 18))  # in inches
    for i, label in enumerate(labels):
        x, y = low_dim_embs[i, :]
        plt.scatter(x, y, s=1.0)
        plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords='offset points',
                 ha='right',
                 va='bottom')
    plt.show()
#     plt.savefig(filename)

# Creating the tsne plot [Warning: will take time]
tsne = TSNE(perplexity=30.0, n_components=2, init='pca', n_iter=5000)

low_dim_embedding = tsne.fit_transform(np.array([word.vector for word in nouns]))

# Finally plotting and saving the fig 
plot_with_labels(low_dim_embedding, [word.text for word in nouns])

"""
Explanation: Let's look at some real world data
These is are the last 100 news articles at clarin.com, the biggest news media outlet in Argentina
End of explanation
"""


