csvfile = open(
reader = 
data = []
for line in reader:
    line[3] = line[3].decode('utf-8')
    

# getting the number of rows

#taking a look at the first row

comment_text = 

"""
Explanation: Getting data into Python (basic python i/o)
End of explanation
"""


comment_text

# strings are like lists of characters

# use a colon for start:end indexes

# they can be stuck together easily

# and split apart
comment_text

split_on_questions = 

# it's easy to strip whitespace off of them
for string in split_on_questions:
    

# and cast them to one case
cleaned = 
cleaned

# join them back together

# and look for substring inside them

"""
Explanation: Basic python string manipulation
End of explanation
"""


for row in data:
    comment_text = row[-1]

"""
Explanation: CHALLENGE: count the number of times the words "Hilary" or "Clinton" appear in the dataset
End of explanation
"""


blob = TextBlob(data[80][-1])
blob

# we can get lists of sentences

# lists of words

# lists of "tokens" (punctuation included)

# even parts of speech and noun phrases


"""
Explanation: Introducing TextBlob
Like a supercharged string, with lots of NLP niceties
End of explanation
"""


word_count = collections.Counter(

word_count

"""
Explanation: Summarizing/keywording text
How might we find representative words or phrases of a document?
A place to start: which words appear at the highest frequency in this document?
End of explanation
"""


stopwords = nltk.corpus.stopwords.words('english')

nltk.download()

for key in word_count.keys():
    

"""
Explanation: Challenge: get overall word counts for all comments combined
potential approaches:
* glue together all comments into one big blob
* get word counts for each comment individually and use Counter's update function
The Problem: words we use frequently don't make good unique identifiers.
One solution: use a list of words we don't want to include
"Stop Words"
End of explanation
"""


N_documents = float(len(data))
word_document_counts = 
word_idf = {}

for row in data[1:]:
    blob = TextBlob(row[-1].lower())
    

# calculate IDFs


"""
Explanation: We could continue to add on stopwords as we try to make these keywords better. But it's kind of like playing whack-a-mole
An additional solution to The Problem: add a new term to our "representative-ness" measure that accounts for the overall rarity of the word
$$\frac { { n }_{ w } }{ N }  $$
where ${ n }_{ w }$ is the number of documents containing word $ w $, and $ N $ is the total number of documents.
But we want a potential keyword to have a lower score if it is common in the corpus and a higher score if it is rarer, so we flip it:
$$\frac { N }{ { n }_{ w } }  $$
It's also common to take the log of this to reduce the amount of disparity between extremely common and extremely uncommon terms.
$$ \log \frac { N }{ { n }_{ w } }  $$
This is called IDF, or Inverse Document Frequency. Let's calculate it for all the words in our comment dataset!
End of explanation
"""


comment = data[80][-1]
blob = TextBlob(comment.lower())
num_words_in_comment = len(blob.words)
word_count = blob.word_counts

tf_scores = {}
for word, count in word_count.iteritems():
    if word not in stopwords and len(word) > 2:
        tf_scores[word] = 

tf_idf = {}
for word, tf in tf_scores.iteritems():
    tf_idf[word] = 

sorted(tf_idf.iteritems(), key=lambda k: k[1], reverse=True)[:5]

"""
Explanation: For each word $ w $ in a given document $ D $, we can multiply the term frequency 
$$\frac { { D }{ w } }{ { W }{ D } } $$ 
where $ { D }_{ w } $ is the number of occurrences of word $ w $ in document $ D $
and $ { W }_{ D } $ is the total number of words in document $ D $
with the word's IDF that we just calculated to get TF-IDF scores, the highest ones being words that likely to be good representatives of that document.
End of explanation
"""


from nltk.stem.porter import PorterStemmer

stemmer = PorterStemmer()
print stemmer.stem('political')
print stemmer.stem('politics')
print stemmer.stem('politician')

"""
Explanation: Note that TF-IDF can be tweaked in lots of other ways if you aren't getting good results.
It can also be done with "n-grams"— phrases that are n words long to capture multi word phrases like "gay rights" or "hillary clinton"
Additional demonstrations
Boiling down words: stemming
End of explanation
"""


from nltk.text import Text
tokens = TextBlob(data[80][-1]).tokens
text_object = Text(tokens)
text_object.concordance('Hilary')

"""
Explanation: Seeing words in context: concordance
End of explanation
"""


blob = TextBlob(data[41][-1])
blob

blob.sentiment

blob.sentences[1].sentiment

"""
Explanation: Sentiment Analysis
End of explanation
"""


