# Tensorflow
import tensorflow as tf
print('Tested with TensorFLow 1.2.0')
print('Your TensorFlow version:', tf.__version__) 

# Feeding function for enqueue data
from tensorflow.python.estimator.inputs.queues import feeding_functions as ff

# Rnn common functions
from tensorflow.contrib.learn.python.learn.estimators import rnn_common

# Model builder
from tensorflow.python.estimator import model_fn as model_fn_lib

# Run an experiment
from tensorflow.contrib.learn.python.learn import learn_runner

# Helpers for data processing
import pandas as pd
import numpy as np
import argparse
import random

"""
Explanation: Dependencies
End of explanation
"""


# data from: http://ai.stanford.edu/~amaas/data/sentiment/
TRAIN_INPUT = 'data/train.csv'
TEST_INPUT = 'data/test.csv'

# data manually generated
MY_TEST_INPUT = 'data/mytest.csv'

# wordtovec
# https://nlp.stanford.edu/projects/glove/
# the matrix will contain 400,000 word vectors, each with a dimensionality of 50.
word_list = np.load('word_list.npy')
word_list = word_list.tolist() # originally loaded as numpy array
word_list = [word.decode('UTF-8') for word in word_list] # encode words as UTF-8
print('Loaded the word list, length:', len(word_list))

word_vector = np.load('word_vector.npy')
print ('Loaded the word vector, shape:', word_vector.shape)

"""
Explanation: Loading Data
First, we want to create our word vectors. For simplicity, we're going to be using a pretrained model. 
As one of the biggest players in the ML game, Google was able to train a Word2Vec model on a massive Google News dataset that contained over 100 billion different words! From that model, Google was able to create 3 million word vectors, each with a dimensionality of 300. 
In an ideal scenario, we'd use those vectors, but since the word vectors matrix is quite large (3.6 GB!), we'll be using a much more manageable matrix that is trained using GloVe, a similar word vector generation model. The matrix will contain 400,000 word vectors, each with a dimensionality of 50. 
We're going to be importing two different data structures, one will be a Python list with the 400,000 words, and one will be a 400,000 x 50 dimensional embedding matrix that holds all of the word vector values.
End of explanation
"""


baseball_index = word_list.index('baseball')
print('Example: baseball')
print(word_vector[baseball_index])

"""
Explanation: We can also search our word list for a word like "baseball", and then access its corresponding vector through the embedding matrix.
End of explanation
"""


max_seq_length = 10 # maximum length of sentence
num_dims = 300 # dimensions for each word vector

first_sentence = np.zeros((max_seq_length), dtype='int32')
first_sentence[0] = word_list.index("i")
first_sentence[1] = word_list.index("thought")
first_sentence[2] = word_list.index("the")
first_sentence[3] = word_list.index("movie")
first_sentence[4] = word_list.index("was")
first_sentence[5] = word_list.index("incredible")
first_sentence[6] = word_list.index("and")
first_sentence[7] = word_list.index("inspiring")
# first_sentence[8] = 0
# first_sentence[9] = 0

print(first_sentence.shape)
print(first_sentence) # shows the row index for each word

"""
Explanation: Now that we have our vectors, our first step is taking an input sentence and then constructing the its vector representation. Let's say that we have the input sentence "I thought the movie was incredible and inspiring". In order to get the word vectors, we can use Tensorflow's embedding lookup function. This function takes in two arguments, one for the embedding matrix (the wordVectors matrix in our case), and one for the ids of each of the words. The ids vector can be thought of as the integerized representation of the training set. This is basically just the row index of each of the words. Let's look at a quick example to make this concrete.
End of explanation
"""


with tf.Session() as sess:
    print(tf.nn.embedding_lookup(word_vector, first_sentence).eval().shape)

"""
Explanation: TODO### Insert image
The 10 x 50 output should contain the 50 dimensional word vectors for each of the 10 words in the sequence.
End of explanation
"""


from os import listdir
from os.path import isfile, join
positiveFiles = ['positiveReviews/' + f for f in listdir('positiveReviews/') if isfile(join('positiveReviews/', f))]
negativeFiles = ['negativeReviews/' + f for f in listdir('negativeReviews/') if isfile(join('negativeReviews/', f))]
numWords = []
for pf in positiveFiles:
    with open(pf, "r", encoding='utf-8') as f:
        line=f.readline()
        counter = len(line.split())
        numWords.append(counter)       
print('Positive files finished')

for nf in negativeFiles:
    with open(nf, "r", encoding='utf-8') as f:
        line=f.readline()
        counter = len(line.split())
        numWords.append(counter)  
print('Negative files finished')

numFiles = len(numWords)
print('The total number of files is', numFiles)
print('The total number of words in the files is', sum(numWords))
print('The average number of words in the files is', sum(numWords)/len(numWords))

"""
Explanation: Before creating the ids matrix for the whole training set, let’s first take some time to visualize the type of data that we have. This will help us determine the best value for setting our maximum sequence length. In the previous example, we used a max length of 10, but this value is largely dependent on the inputs you have.  
The training set we're going to use is the Imdb movie review dataset. This set has 25,000 movie reviews, with 12,500 positive reviews and 12,500 negative reviews. Each of the reviews is stored in a txt file that we need to parse through. The positive reviews are stored in one directory and the negative reviews are stored in another. The following piece of code will determine total and average number of words in each review.
End of explanation
"""


import matplotlib.pyplot as plt
%matplotlib inline
plt.hist(numWords, 50)
plt.xlabel('Sequence Length')
plt.ylabel('Frequency')
plt.axis([0, 1200, 0, 8000])
plt.show()

"""
Explanation: We can also use the Matplot library to visualize this data in a histogram format.
End of explanation
"""


max_seq_len = 250

"""
Explanation: From the histogram as well as the average number of words per file, we can safely say that most reviews will fall under 250 words, which is the max sequence length value we will set.
End of explanation
"""


ids_matrix = np.load('ids_matrix.npy').tolist()

"""
Explanation: Data
End of explanation
"""


# Parameters for training
STEPS = 100000
BATCH_SIZE = 32

# Parameters for data processing
REVIEW_KEY = 'review'
SEQUENCE_LENGTH_KEY = 'sequence_length'

"""
Explanation: Parameters
End of explanation
"""


POSITIVE_REVIEWS = 12500

# copying sequences
data_sequences = [np.asarray(v, dtype=np.int32) for v in ids_matrix]
# generating labels
data_labels = [[1, 0] if i < POSITIVE_REVIEWS else [0, 1] for i in range(len(ids_matrix))]
# also creating a length column, this will be used by the Dynamic RNN
# see more about it here: https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn
data_length = [max_seq_len for i in range(len(ids_matrix))]

"""
Explanation: Separating train and test data
The training set we're going to use is the Imdb movie review dataset. This set has 25,000 movie reviews, with 12,500 positive reviews and 12,500 negative reviews. 
Let's first give a positive label [1, 0] to the first 12500 reviews, and a negative label [0, 1] to the other reviews.
End of explanation
"""


data = list(zip(data_sequences, data_labels))
random.shuffle(data) # shuffle

data = np.asarray(data)
print(data.shape)
# separating train and test data
limit = int(len(data) * 0.9)

train_data = data[:limit]
test_data = data[limit:]

"""
Explanation: Then, let's shuffle the data and use 90% of the reviews for training and the other 10% for testing.
End of explanation
"""


LABEL_INDEX = 1
def _number_of_pos_labels(df):
    pos_labels = 0
    for value in df:
        if value[LABEL_INDEX] == [1, 0]:
            pos_labels += 1
    return pos_labels

pos_labels_train = _number_of_pos_labels(train_data)
total_labels_train = len(train_data)

pos_labels_test = _number_of_pos_labels(test_data)
total_labels_test = len(test_data)

print('Total number of positive labels:', pos_labels_train + pos_labels_test)
print('Proportion of positive labels on the Train data:', pos_labels_train/total_labels_train)
print('Proportion of positive labels on the Test data:', pos_labels_test/total_labels_test)

"""
Explanation: Verifying if the train and test data have enough positive and negative examples
End of explanation
"""


def get_input_fn(df, batch_size, num_epochs=1, shuffle=True):  
    def input_fn():
        # https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/data
        sequences = np.asarray([v for v in df[:,0]], dtype=np.int32)
        labels = np.asarray([v for v in df[:,1]], dtype=np.int32)
        length = np.asarray(df[:,2], dtype=np.int32)
        # 
        dataset = (
            tf.contrib.data.Dataset.from_tensor_slices((sequences, labels, length)) # reading data from memory
            .repeat(num_epochs) # repeat dataset the number of epochs
            .batch(batch_size)
        )
        
        # for our "manual" test we don't want to shuffle the data
        if shuffle:
            dataset = dataset.shuffle(buffer_size=100000)

        # create iterator
        review, label, length = dataset.make_one_shot_iterator().get_next()

        features = {
            REVIEW_KEY: review,
            SEQUENCE_LENGTH_KEY: length,
        }

        return features, label
    return input_fn

features, label = get_input_fn(train_data, 2)()

with tf.Session() as sess:
    items = sess.run(features)
    print(items[REVIEW_KEY])
    print

    items = sess.run(features)
    print(items[REVIEW_KEY])
    print

train_input_fn = get_input_fn(train_data, BATCH_SIZE, None)
test_input_fn = get_input_fn(test_data, BATCH_SIZE)

"""
Explanation: Input functions
End of explanation
"""


def get_model_fn(rnn_cell_sizes,
                 label_dimension,
                 dnn_layer_sizes=[],
                 optimizer='SGD',
                 learning_rate=0.01,
                 embed_dim=128):
    
    def model_fn(features, labels, mode):
        
        review = features[REVIEW_KEY]

        # Creating dense representation for the sentences
        # and then converting it to embeding representation
        data = tf.nn.embedding_lookup(word_vector, review)
        
        # Each RNN layer will consist of a LSTM cell
        #rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in rnn_cell_sizes]
        lstm_cell = tf.nn.rnn_cell.LSTMCell(64)
        lstm_cell = tf.nn.rnn_cell.DropoutWrapper(cell=lstm_cell, output_keep_prob=0.25)
        
        # Construct the layers
        #multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
        
        # Runs the RNN model dynamically
        # more about it at: 
        # https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn
        outputs, final_state = tf.nn.dynamic_rnn(cell=lstm_cell,
                                                 inputs=data,
                                                 dtype=tf.float32)

        # Slice to keep only the last cell of the RNN
        last_activations = rnn_common.select_last_activations(outputs,
                                                              sequence_length)

        # Final dense layer for prediction
        predictions = tf.layers.dense(last_activations, label_dimension)
        predictions_softmax = tf.nn.softmax(predictions)
        
        loss = None
        train_op = None
        
        preds_op = {
            'prediction': predictions_softmax,
            'label': labels
        }
        
        eval_op = {
            "accuracy": tf.metrics.accuracy(
                     tf.argmax(input=predictions_softmax, axis=1),
                     tf.argmax(input=labels, axis=1))
        }
        
        if mode != tf.estimator.ModeKeys.PREDICT:    
            loss = tf.losses.softmax_cross_entropy(labels, predictions)
    
        if mode == tf.estimator.ModeKeys.TRAIN:    
            train_op = tf.contrib.layers.optimize_loss(
              loss,
              tf.contrib.framework.get_global_step(),
              optimizer=optimizer,
              learning_rate=learning_rate)
        
        return tf.contrib.learn.ModelFnOps(mode,
                                           predictions=predictions_softmax,
                                           loss=loss,
                                           train_op=train_op,
                                           eval_metric_ops=eval_op)
    return model_fn

model_fn = get_model_fn(rnn_cell_sizes=[64], # size of the hidden layers
                        label_dimension=2, # since are just 2 classes
                        dnn_layer_sizes=[128, 64], # size of units in the dense layers on top of the RNN
                        optimizer='Adam',
                        learning_rate=0.001,
                        embed_dim=512)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn, model_dir='tensorboard18/')

"""
Explanation: Creating the Estimator model
End of explanation
"""


# create experiment
def generate_experiment_fn():
  
  """
  Create an experiment function given hyperparameters.
  Returns:
    A function (output_dir) -> Experiment where output_dir is a string
    representing the location of summaries, checkpoints, and exports.
    this function is used by learn_runner to create an Experiment which
    executes model code provided in the form of an Estimator and
    input functions.
    All listed arguments in the outer function are used to create an
    Estimator, and input functions (training, evaluation, serving).
    Unlisted args are passed through to Experiment.
  """

  def _experiment_fn(output_dir):
    return tf.contrib.learn.Experiment(
        estimator,
        train_input_fn=train_input_fn,
        eval_input_fn=test_input_fn,
        train_steps=STEPS
    )
  return _experiment_fn

# run experiment 
learn_runner.run(generate_experiment_fn(), '/tmp/outputdir')

"""
Explanation: Create and Run Experiment
End of explanation
"""


preds = estimator.predict(input_fn=my_test_input_fn, as_iterable=True)

sentences = _get_csv_column(MY_TEST_INPUT, 'review')

print()
for p, s in zip(preds, sentences):
    print('sentence:', s)
    print('bad review:', p[0], 'good review:', p[1])
    print('-' * 10)

"""
Explanation: Making Predictions
End of explanation
"""


