def pretty_print_review_and_label(i):
    print(labels[i] + "\t:\t" + reviews[i][:80] + "...")

g = open('reviews.txt','r') # What we know!
reviews = list(map(lambda x:x[:-1],g.readlines()))
g.close()

g = open('labels.txt','r') # What we WANT to know!
labels = list(map(lambda x:x[:-1].upper(),g.readlines()))
g.close()

len(reviews)

reviews[0]

labels[0]

"""
Explanation: Sentiment Classification & How To "Frame Problems" for a Neural Network
by Andrew Trask

Twitter: @iamtrask
Blog: http://iamtrask.github.io

What You Should Already Know

neural networks, forward and back-propagation
stochastic gradient descent
mean squared error
and train/test splits

Where to Get Help if You Need it

Re-watch previous Udacity Lectures
Leverage the recommended Course Reading Material - Grokking Deep Learning (40% Off: traskud17)
Shoot me a tweet @iamtrask

Tutorial Outline:


Intro: The Importance of "Framing a Problem"


Curate a Dataset

Developing a "Predictive Theory"

PROJECT 1: Quick Theory Validation


Transforming Text to Numbers


PROJECT 2: Creating the Input/Output Data


Putting it all together in a Neural Network


PROJECT 3: Building our Neural Network


Understanding Neural Noise


PROJECT 4: Making Learning Faster by Reducing Noise


Analyzing Inefficiencies in our Network


PROJECT 5: Making our Network Train and Run Faster


Further Noise Reduction


PROJECT 6: Reducing Noise by Strategically Reducing the Vocabulary


Analysis: What's going on in the weights?


Lesson: Curate a Dataset
End of explanation
"""


print("labels.txt \t : \t reviews.txt\n")
pretty_print_review_and_label(2137)
pretty_print_review_and_label(12816)
pretty_print_review_and_label(6267)
pretty_print_review_and_label(21934)
pretty_print_review_and_label(5297)
pretty_print_review_and_label(4998)

"""
Explanation: Lesson: Develop a Predictive Theory
End of explanation
"""


from collections import Counter
import numpy as np

positive_counts = Counter()
negative_counts = Counter()
total_counts = Counter()

for i in range(len(reviews)):
    if(labels[i] == 'POSITIVE'):
        for word in reviews[i].split(" "):
            positive_counts[word] += 1
            total_counts[word] += 1
    else:
        for word in reviews[i].split(" "):
            negative_counts[word] += 1
            total_counts[word] += 1

positive_counts.most_common()

pos_neg_ratios = Counter()

for term,cnt in list(total_counts.most_common()):
    if(cnt > 100):
        pos_neg_ratio = positive_counts[term] / float(negative_counts[term]+1)
        pos_neg_ratios[term] = pos_neg_ratio

for word,ratio in pos_neg_ratios.most_common():
    if(ratio > 1):
        pos_neg_ratios[word] = np.log(ratio)
    else:
        pos_neg_ratios[word] = -np.log((1 / (ratio+0.01)))

# words most frequently seen in a review with a "POSITIVE" label
pos_neg_ratios.most_common()

# words most frequently seen in a review with a "NEGATIVE" label
list(reversed(pos_neg_ratios.most_common()))[0:30]

"""
Explanation: Project 1: Quick Theory Validation
End of explanation
"""


from IPython.display import Image

review = "This was a horrible, terrible movie."

Image(filename='sentiment_network.png')

review = "The movie was excellent"

Image(filename='sentiment_network_pos.png')

"""
Explanation: Transforming Text into Numbers
End of explanation
"""


vocab = set(total_counts.keys())
vocab_size = len(vocab)
print(vocab_size)

list(vocab)

import numpy as np

layer_0 = np.zeros((1,vocab_size))
layer_0

from IPython.display import Image
Image(filename='sentiment_network.png')

word2index = {}

for i,word in enumerate(vocab):
    word2index[word] = i
word2index

def update_input_layer(review):
    
    global layer_0
    
    # clear out previous state, reset the layer to be all 0s
    layer_0 *= 0
    for word in review.split(" "):
        layer_0[0][word2index[word]] += 1

update_input_layer(reviews[0])

layer_0

y = [1 if l=="POSITIVE" else 0 for l in labels]
y[:10]

def get_target_for_label(label):
    if(label == 'POSITIVE'):
        return 1
    else:
        return 0

labels[0]

get_target_for_label(labels[0])

labels[1]

get_target_for_label(labels[1])

"""
Explanation: Project 2: Creating the Input/Output Data
End of explanation
"""


a = [[4,5]]
print(a)
a.append([6])
a

x = []
for review in reviews:
    update_input_layer(review)
    x.append(layer_0)
x[:5]

x = np.array(x)
x[:5]

"""
Explanation: Project 3: Building a Neural Network

Start with your neural network from the last chapter
3 layer neural network
no non-linearity in hidden layer
use our functions to create the training data
create a "pre_process_data" function to create vocabulary for our training data generating functions
modify "train" to train over the entire corpus

Where to Get Help if You Need it

Re-watch previous week's Udacity Lectures
Chapters 3-5 - Grokking Deep Learning - (40% Off: traskud17)

Generating the dataset
End of explanation
"""


y = [get_target_for_label(l) for l in labels] 
for i in range(10):
    print(labels[i], y[i])
y[:20]

import time
import sys
import numpy as np

class SentimentNetwork(object):
    def __init__(self, reviews, labels, hidden_nodes=10, output_nodes=1, learning_rate=0.1):
        # set our random number generator 
        np.random.seed(1)
    
        self.pre_process_data(reviews, labels)
        
        self.init_network(len(self.review_vocab),hidden_nodes, 1, learning_rate)
        
        print('NeuralNet intiated!')
        print('Nodes:', self.input_nodes, self.hidden_nodes, self.output_nodes)
        print('learning rate: ', self.learning_rate)

        
    def init_network(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
        # Set number of nodes in input, hidden and output layers.
        self.input_nodes = input_nodes
        self.hidden_nodes = hidden_nodes
        self.output_nodes = output_nodes

        
        # Initialize weights
        self.weights_0_1 = np.zeros((self.input_nodes,self.hidden_nodes))
    
        self.weights_1_2 = np.random.normal(0.0, self.output_nodes**-0.5, 
                                                (self.hidden_nodes, self.output_nodes))
        
        self.learning_rate = learning_rate
        self.layer_0 = np.zeros((1,input_nodes))
        
    def sigmoid(self, x):
        return 1 / (1 + np.exp(-x))
        
    def sigmoid_output_2_derivative(self,output):
        return output * (1 - output)
    
    def pre_process_data(self, reviews, labels):
        assert len(reviews) == len(labels)
        
        #self.y = [1 if label=="POSITIVE" else 0 for label in labels]
        
        review_vocab = set()
        for review in reviews:
            for word in review.split(" "):
                review_vocab.add(word)
        self.review_vocab = list(review_vocab)
        
        label_vocab = set()
        for label in labels:
            label_vocab.add(label)
        
        self.label_vocab = list(label_vocab)
        
        self.review_vocab_size = len(self.review_vocab)
        self.label_vocab_size = len(self.label_vocab)
        
        self.word2index = {}
        for i, word in enumerate(self.review_vocab):
            self.word2index[word] = i
        
        self.label2index = {}
        for i, label in enumerate(self.label_vocab):
            self.label2index[label] = i
        
        
    def get_target_for_label(self, label):
        if(label == 'POSITIVE'):
            return 1
        else:
            return 0
    
    def update_input_layer(self,review):

        # clear out previous state, reset the layer to be all 0s
        self.layer_0 *= 0
        for word in review.split(" "):
            if(word in self.word2index.keys()):
                self.layer_0[0][self.word2index[word]] = 1
                
        
    def train(self, inputs_list, targets_list):
        assert len(inputs_list) == len(targets_list)
        correct_so_far = 0
        start = time.time()
        
        for i in range(len(inputs_list)):
            review = inputs_list[i]
            label = targets_list[i]
            label_value = self.get_target_for_label(label)
            
            # Convert inputs list to 2d array
            #inputs = np.array(inputs_list, ndmin=2).T
            #targets = np.array(targets_list, ndmin=2).T

            #### Implement the forward pass here ####
            ### Forward pass ###
            
            self.update_input_layer(review)
            
            # input layer
            layer_1 = np.dot(self.layer_0, self.weights_0_1)
            
            # output layer
            layer_2 = self.sigmoid(np.dot(layer_1, self.weights_1_2))
            
            
            #### Implement the backward pass here ####
            ### Backward pass ###

            # TODO: Output error
            layer_2_errors = layer_2 # Output layer error is the difference between desired target and actual output.
            layer_2_delta = layer_2_errors * self.sigmoid_output_2_derivative(layer_2)

            # TODO: Backpropagated error
            layer_1_errors = np.dot(layer_2_delta, self.weights_1_2.T) # errors propagated to the hidden layer
            layer_1_delta = layer_1_errors # hidden layer gradients
            
            # TODO: Update the weights
            self.weights_1_2 -= layer_1.T.dot(layer_2_delta) * self.learning_rate # update hidden-to-output weights with gradient descent step
            self.weights_0_1 -= self.layer_0.T.dot(layer_1_delta) * self.learning_rate # update input-to-hidden weights with gradient descent step
         
            if(np.abs(layer_2_errors) < 0.5):
                correct_so_far += 1
            
            reviews_per_second = i / float(time.time() - start)
            
            sys.stdout.write("\rProgress:" + str(100 * i/float(len(inputs_list)))[:4] + "% Speed(reviews/sec):" + str(reviews_per_second)[0:5] + " #Correct:" + str(correct_so_far) + " #Trained:" + str(i+1) + " Training Accuracy:" + str(correct_so_far * 100 / float(i+1))[:4] + "%")
            if(i % 2500 == 0):
                print("")
    
    def test(self, testing_reviews, testing_labels):
        
        correct = 0
        
        start = time.time()
        
        for i in range(len(testing_reviews)):
            pred = self.run(testing_reviews[i])
            if(pred == testing_labels[i]):
                correct += 1
            
            reviews_per_second = i / float(time.time() - start)
            
            sys.stdout.write("\rProgress:" + str(100 * i/float(len(testing_reviews)))[:4] \
                             + "% Speed(reviews/sec):" + str(reviews_per_second)[0:5] \
                            + "% #Correct:" + str(correct) + " #Tested:" + str(i+1) + " Testing Accuracy:" + str(correct * 100 / float(i+1))[:4] + "%")
    
                  
    def run(self, review):
        
        # Input Layer
        self.update_input_layer(review.lower())

        # Hidden layer
        layer_1 = self.layer_0.dot(self.weights_0_1)

        # Output layer
        layer_2 = self.sigmoid(layer_1.dot(self.weights_1_2))
        
        if(layer_2[0] > 0.5):
            return "POSITIVE"
        else:
            return "NEGATIVE"
    
def MSE(y, Y):
    return np.mean((y-Y)**2)

### Set the hyperparameters here ###
epochs = 100
learning_rate = 0.04
hidden_nodes = 2
output_nodes = 1

mlp = SentimentNetwork(reviews[:-1000],labels[:-1000], learning_rate=1.0)

print(mlp.weights_0_1)
print(mlp.weights_1_2)

# evaluate our model before training (just to show how horrible it is)
mlp.test(reviews[-1000:],labels[-1000:])

# train the network
print(mlp.learning_rate)
mlp.train(reviews[:-1000],labels[:-1000])

print(mlp.weights_0_1)
print(mlp.weights_1_2)
mlp.test(reviews[-1000:],labels[-1000:])

"""
Explanation: The target dataset
End of explanation
"""


