def pretty_print_review_and_label(i):
    print(labels[i] + "\t:\t" + reviews[i][:80] + "...")

g = open('reviews.txt','r') # What we know!
reviews = list(map(lambda x:x[:-1],g.readlines()))
g.close()

g = open('labels.txt','r') # What we WANT to know!
labels = list(map(lambda x:x[:-1].upper(),g.readlines()))
g.close()

len(reviews)

reviews[0]

labels[0]

"""
Explanation: Sentiment Classification & How To "Frame Problems" for a Neural Network
by Andrew Trask

Twitter: @iamtrask
Blog: http://iamtrask.github.io

What You Should Already Know

neural networks, forward and back-propagation
stochastic gradient descent
mean squared error
and train/test splits

Where to Get Help if You Need it

Re-watch previous Udacity Lectures
Leverage the recommended Course Reading Material - Grokking Deep Learning (40% Off: traskud17)
Shoot me a tweet @iamtrask

Tutorial Outline:


Intro: The Importance of "Framing a Problem"


Curate a Dataset

Developing a "Predictive Theory"

PROJECT 1: Quick Theory Validation


Transforming Text to Numbers


PROJECT 2: Creating the Input/Output Data


Putting it all together in a Neural Network


PROJECT 3: Building our Neural Network


Understanding Neural Noise


PROJECT 4: Making Learning Faster by Reducing Noise


Analyzing Inefficiencies in our Network


PROJECT 5: Making our Network Train and Run Faster


Further Noise Reduction


PROJECT 6: Reducing Noise by Strategically Reducing the Vocabulary


Analysis: What's going on in the weights?


Lesson: Curate a Dataset
End of explanation
"""


print("labels.txt \t : \t reviews.txt\n")
pretty_print_review_and_label(2137)
pretty_print_review_and_label(12816)
pretty_print_review_and_label(6267)
pretty_print_review_and_label(21934)
pretty_print_review_and_label(5297)
pretty_print_review_and_label(4998)

"""
Explanation: Lesson: Develop a Predictive Theory
End of explanation
"""


from collections import Counter
import numpy as np

positive_counts = Counter()
negative_counts = Counter()
total_counts = Counter()

for i in range(len(reviews)):
    if(labels[i] == 'POSITIVE'):
        for word in reviews[i].split(" "):
            positive_counts[word] += 1
            total_counts[word] += 1
    else:
        for word in reviews[i].split(" "):
            negative_counts[word] += 1
            total_counts[word] += 1

positive_counts.most_common()

pos_neg_ratios = Counter()

for term,cnt in list(total_counts.most_common()):
    if(cnt > 100):
        pos_neg_ratio = positive_counts[term] / float(negative_counts[term]+1)
        pos_neg_ratios[term] = pos_neg_ratio

for word,ratio in pos_neg_ratios.most_common():
    if(ratio > 1):
        pos_neg_ratios[word] = np.log(ratio)
    else:
        pos_neg_ratios[word] = -np.log((1 / (ratio+0.01)))

# words most frequently seen in a review with a "POSITIVE" label
pos_neg_ratios.most_common()

# words most frequently seen in a review with a "NEGATIVE" label
list(reversed(pos_neg_ratios.most_common()))[0:30]

"""
Explanation: Project 1: Quick Theory Validation
End of explanation
"""


