import graphlab

"""
Explanation: Document retrieval from wikipedia data Using TF-IDF
Fire up GraphLab Create
End of explanation
"""


people = graphlab.SFrame('people_wiki.gl/')

"""
Explanation: Load some text data - from wikipedia, pages on people
End of explanation
"""


people.head(5)

len(people)

"""
Explanation: Let's View data set with head to view some of the rows let's 5 rows
Data contains:  link to wikipedia article, name of person, text of article.
End of explanation
"""


obama = people[people['name'] == 'Barack Obama']

"""
Explanation: Explore the dataset and checkout the text it contains
Exploring the entry for president Obama, Let's take subset of data as obama
End of explanation
"""


obama

obama['text']

"""
Explanation: obama variable contain all the information about Barack Obama from wiki
End of explanation
"""


clooney = people[people['name'] == 'George Clooney']
clooney['text']

"""
Explanation: Exploring the entry for actor George Clooney, in variable clooney
End of explanation
"""


obama['word_count'] = graphlab.text_analytics.count_words(obama['text'])

print obama['word_count']

"""
Explanation: Get the word counts for Obama article and store result in same data set as a new variable name word_count
End of explanation
"""


obama_word_count_table = obama[['word_count']].stack('word_count', new_column_name = ['word','count'])

"""
Explanation: Sort the word counts for the Obama article, for sorting we use stack with new column name as word and count
Turning dictonary of word counts into a table
End of explanation
"""


obama_word_count_table.head(5)

obama_word_count_table.sort('count',ascending=False)

"""
Explanation: Sorting the word counts to show most common words at the top
End of explanation
"""


people['word_count'] = graphlab.text_analytics.count_words(people['text'])
people.head(5)

tfidf = graphlab.text_analytics.tf_idf(people['word_count'])
tfidf

"""
Explanation: Most common words include uninformative words like "the", "in", "and",...but some of the words does't contain any meaning full informaion about these article we can remove these words some time we call these words as stop words
Compute TF-IDF for the corpus
To give more weight to informative words, we weigh them by their TF-IDF scores. TF-IDF basically way to score the importance of words in document based on how frequently they appear across multiple doc, but this will work for the whole corpus so we apply tf-idf to the whole
End of explanation
"""


people['tfidf'] = tfidf['docs']

"""
Explanation: add tf-idf docs to people dataset with a new variable called tfidf
End of explanation
"""


obama = people[people['name'] == 'Barack Obama']

obama[['tfidf']].stack('tfidf',new_column_name=['word','tfidf']).sort('tfidf',ascending=False)

"""
Explanation: Examine the TF-IDF for the Obama article, this new variable with tfidf
End of explanation
"""


clinton = people[people['name'] == 'Bill Clinton']

beckham = people[people['name'] == 'David Beckham']

"""
Explanation: Words with highest TF-IDF are much more informative. so we sorted words according to the TF-IDF
Manually compute distances between a few people
Let's manually compare the distances between the articles for a few famous people.
End of explanation
"""


graphlab.distances.cosine(obama['tfidf'][0],clinton['tfidf'][0])

graphlab.distances.cosine(obama['tfidf'][0],beckham['tfidf'][0])

"""
Explanation: Is Obama closer to Clinton than to Beckham?
We will use cosine distance, which is given by
(1-cosine_similarity) for computing the distance between the two documents
and find that the article about president Obama is closer to the one about former president Clinton than that of footballer David Beckham.
End of explanation
"""


knn_model = graphlab.nearest_neighbors.create(people,features=['tfidf'],label='name')

"""
Explanation: less the distance between two document more they are similar two each other like obam and clinton are more similar then obama and beckham
Build a nearest neighbor model for document retrieval
We now create a nearest-neighbors model and apply it to document retrieval.
End of explanation
"""


knn_model.query(obama)

"""
Explanation: Applying the nearest-neighbors model for retrieval
Who is closest to Obama? for that we use query with model
End of explanation
"""


swift = people[people['name'] == 'Taylor Swift']

knn_model.query(swift)

jolie = people[people['name'] == 'Angelina Jolie']

knn_model.query(jolie)

arnold = people[people['name'] == 'Arnold Schwarzenegger']

knn_model.query(arnold)

"""
Explanation: As we can see, president Obama's article is closest to the one about his vice-president Biden, and those of other politicians.  
Other examples of document retrieval
End of explanation
"""


