def pretty_print_review_and_label(i):
    print(labels[i] + "\t: " + reviews[i][:70] + "...")

g = open('reviews.txt','r') # What we know!
reviews = list(map(lambda x:x[:-1],g.readlines()))
g.close()

g = open('labels.txt','r') # What we WANT to know!
labels = list(map(lambda x:x[:-1].upper(),g.readlines()))
g.close()

len(reviews)

reviews[1]

labels[1]

"""
Explanation: Sentiment Classification & How To "Frame Problems" for a Neural Network
by Andrew Trask

Twitter: @iamtrask
Blog: http://iamtrask.github.io

What You Should Already Know

neural networks, forward and back-propagation
stochastic gradient descent
mean squared error
and train/test splits

Where to Get Help if You Need it

Re-watch previous Udacity Lectures
Leverage the recommended Course Reading Material - Grokking Deep Learning (40% Off: traskud17)
Shoot me a tweet @iamtrask

Tutorial Outline:


Intro: The Importance of "Framing a Problem"


Curate a Dataset

Developing a "Predictive Theory"

PROJECT 1: Quick Theory Validation


Transforming Text to Numbers


PROJECT 2: Creating the Input/Output Data


Putting it all together in a Neural Network


PROJECT 3: Building our Neural Network


Understanding Neural Noise


PROJECT 4: Making Learning Faster by Reducing Noise


Analyzing Inefficiencies in our Network


PROJECT 5: Making our Network Train and Run Faster


Further Noise Reduction


PROJECT 6: Reducing Noise by Strategically Reducing the Vocabulary


Analysis: What's going on in the weights?


Lesson: Curate a Dataset
End of explanation
"""


print("labels.txt \t : \t reviews.txt\n")
pretty_print_review_and_label(2137)
pretty_print_review_and_label(12816)
pretty_print_review_and_label(6267)
pretty_print_review_and_label(21934)
pretty_print_review_and_label(5297)
pretty_print_review_and_label(4998)

"""
Explanation: Lesson: Develop a Predictive Theory
End of explanation
"""


from collections import Counter
import numpy as np

c = Counter()

for review in reviews:
    for word in str(review).split():
        c[word] += 1

"""
Explanation: Counting all the words:
End of explanation
"""


common_words = c.most_common(24)
common_words

for word in common_words:
    del c[word[0]]
c.most_common(10)

"""
Explanation: the most common words have no predictive power:
End of explanation
"""


negative_words = Counter()
positive_words = Counter()
total_words = Counter()

for i, review in enumerate(reviews):
    for word in str(review).split():
        if labels[i] == "NEGATIVE":
            negative_words[word] += 1
        else:
            positive_words[word] += 1
        total_words[word] += 1

pos_neg_ratios = Counter()
neg_pos_ratios = Counter()
for word, cnt in total_words.most_common():
    if cnt > 500:
        pos_neg_ratios[word] += positive_words[word] / (negative_words[word] + 1.0)
        neg_pos_ratios[word] += negative_words[word] / (positive_words[word] + 1.0)
        
pos_neg_ratios.most_common(20)

neg_pos_ratios.most_common(20)

"""
Explanation: Hmmm.. it would be more useful to have two counters, one for negative reviews and the other for positive ones.
End of explanation
"""


