import graphlab

"""
Explanation: Document retrieval from wikipedia data
Fire up GraphLab Create
End of explanation
"""


people = graphlab.SFrame('people_wiki.gl/')

"""
Explanation: Load some text data - from wikipedia, pages on people
End of explanation
"""


people.head()

len(people)

"""
Explanation: Data contains:  link to wikipedia article, name of person, text of article.
End of explanation
"""


obama = people[people['name'] == 'Barack Obama']

obama

obama['text']

"""
Explanation: Explore the dataset and checkout the text it contains
Exploring the entry for president Obama
End of explanation
"""


clooney = people[people['name'] == 'George Clooney']
clooney['text']

"""
Explanation: Exploring the entry for actor George Clooney
End of explanation
"""


obama['word_count'] = graphlab.text_analytics.count_words(obama['text'])

print obama['word_count']

"""
Explanation: Get the word counts for Obama article
End of explanation
"""


obama_word_count_table = obama[['word_count']].stack('word_count', new_column_name = ['word','count'])

"""
Explanation: Sort the word counts for the Obama article
Turning dictonary of word counts into a table
End of explanation
"""


obama_word_count_table.head()

obama_word_count_table.sort('count',ascending=False)

"""
Explanation: Sorting the word counts to show most common words at the top
End of explanation
"""


people['word_count'] = graphlab.text_analytics.count_words(people['text'])
people.head()

tfidf = graphlab.text_analytics.tf_idf(people['word_count'])

# Earlier versions of GraphLab Create returned an SFrame rather than a single SArray
# This notebook was created using Graphlab Create version 1.7.1
if graphlab.version <= '1.6.1':
    tfidf = tfidf['docs']

tfidf

people['tfidf'] = tfidf

people.head()

"""
Explanation: Most common words include uninformative words like "the", "in", "and",...
Compute TF-IDF for the corpus
To give more weight to informative words, we weigh them by their TF-IDF scores.
End of explanation
"""


obama = people[people['name'] == 'Barack Obama']

obama[['tfidf']].stack('tfidf',new_column_name=['word','tfidf']).sort('tfidf',ascending=False)

"""
Explanation: Examine the TF-IDF for the Obama article
End of explanation
"""


clinton = people[people['name'] == 'Bill Clinton']

beckham = people[people['name'] == 'David Beckham']

"""
Explanation: Words with highest TF-IDF are much more informative.
Manually compute distances between a few people
Let's manually compare the distances between the articles for a few famous people.
End of explanation
"""


graphlab.distances.cosine(obama['tfidf'][0],clinton['tfidf'][0])

graphlab.distances.cosine(obama['tfidf'][0],beckham['tfidf'][0])

"""
Explanation: Is Obama closer to Clinton than to Beckham?
We will use cosine distance, which is given by
(1-cosine_similarity) 
and find that the article about president Obama is closer to the one about former president Clinton than that of footballer David Beckham.
End of explanation
"""


knn_model = graphlab.nearest_neighbors.create(people,features=['tfidf'],label='name')

knn_model.summary()

"""
Explanation: Build a nearest neighbor model for document retrieval
We now create a nearest-neighbors model and apply it to document retrieval.
End of explanation
"""


knn_model.query(obama)

"""
Explanation: Applying the nearest-neighbors model for retrieval
Who is closest to Obama?
End of explanation
"""


swift = people[people['name'] == 'Taylor Swift']

knn_model.query(swift)

jolie = people[people['name'] == 'Angelina Jolie']

knn_model.query(jolie)

arnold = people[people['name'] == 'Arnold Schwarzenegger']

knn_model.query(arnold)

elton = people[people['name'] == 'Elton John']

elton

elton[['word_count']].stack('word_count', new_column_name = ['word','count']).sort('count',ascending=False)


elton[['tfidf']].stack('tfidf',new_column_name=['word','tfidf']).sort('tfidf',ascending=False)

victoria = people[people['name'] == 'Victoria Beckham']

graphlab.distances.cosine(elton['tfidf'][0],victoria['tfidf'][0])

paul = people[people['name'] == 'Paul McCartney']

graphlab.distances.cosine(elton['tfidf'][0],paul['tfidf'][0])

knn_model_counts_cosine = graphlab.nearest_neighbors.create(people,features=['word_count'],label='name',distance='cosine')

knn_model_tfidf_cosine = graphlab.nearest_neighbors.create(people,features=['tfidf'],label='name',distance='cosine')

knn_model_counts_cosine.query(elton)

knn_model_tfidf_cosine.query(elton)

knn_model_counts_cosine.query(victoria)

knn_model_tfidf_cosine.query(victoria)

"""
Explanation: As we can see, president Obama's article is closest to the one about his vice-president Biden, and those of other politicians.  
Other examples of document retrieval
End of explanation
"""


