tf.reset_default_graph()

"""
Explanation: Catigorical Columns Don't just One-Hot, They Count.
End of explanation
"""


strings = np.array([['a','a','','b','c'],['a','c','zz','',''],['b','qq','qq','b','']])

"""
Explanation: Here we have 3 examples, each containing 5 strings.

Note: empty strings are ignored, and can be used as padding
End of explanation
"""


vocab = np.array(['a','b','c','<UNK>'],dtype=object)
sparse = tf.feature_column.categorical_column_with_vocabulary_list('strings',vocab, default_value=len(vocab)-1)

"""
Explanation: Now we define a categorical column to represent it.
End of explanation
"""


layer = tf.feature_column.input_layer({'strings':strings}, [tf.feature_column.indicator_column(sparse)])

with tf.train.MonitoredSession() as sess:
    input_value = sess.run(layer)

pd.DataFrame(input_value, columns=vocab)

"""
Explanation: Use indicator_column to define a dense representation, and input_layer to build the conversion operations.
End of explanation
"""


tf.reset_default_graph()

layer = tf.feature_column.input_layer(
    {'strings':strings},
    [tf.feature_column.embedding_column(sparse,10, combiner='mean')])

init = tf.global_variables_initializer()
with tf.train.MonitoredSession() as sess:
    sess.run(init)
    input_value = sess.run(layer)

"""
Explanation: Embedding Columns reduce Over Entries, Using the combiner
End of explanation
"""


input_value.shape

input_value

"""
Explanation: The result is 3, 10d embeddings, because it takes the mean over the strings in each example.
End of explanation
"""


tf.reset_default_graph()

vocab

sparse = tf.contrib.layers.sparse_column_with_keys('strings', vocab)

layer = tf.contrib.layers.sequence_input_from_feature_columns(
    {'strings':tf.constant(strings)},
    [tf.contrib.layers.embedding_column(sparse ,10)])

"""
Explanation: To Skip the reduce?
Use sequence_input_from_feature_columns

Note: this is only compatible with contrib feature_columns
End of explanation
"""


init = tf.global_variables_initializer()
with tf.train.MonitoredSession() as sess:
    sess.run(init)
    input_value = sess.run(layer)

input_value.shape

"""
Explanation: If we run this, it gives a 10d embedding for each of the 5 strings in each of the 3 examples
End of explanation
"""


tf.reset_default_graph()

shape = tf.shape(strings) 
embedding_dim = 10

layer = tf.feature_column.input_layer(
    {'strings':tf.reshape(strings,[tf.reduce_prod(shape)])},
    [tf.feature_column.embedding_column(sparse,embedding_dim, combiner='mean')])

layer = tf.reshape(layer,tf.concat([shape, [embedding_dim]],0))

init = tf.global_variables_initializer()
with tf.train.MonitoredSession() as sess:
    sess.run(init)
    input_value = sess.run(layer)

input_value.shape

"""
Explanation: Or some careful reshaping
End of explanation
"""


import tensorflow as tf
from tensorflow.contrib import keras as keras
import numpy as np

tf.reset_default_graph()

"""
Explanation: Linear Bag of Words Model
End of explanation
"""


NUM_WORDS=1000 # only use top 1000 words
MAX_LEN=250    # truncate after 250 words
INDEX_FROM=3   # word index offset

train,test = keras.datasets.imdb.load_data(maxlen=MAX_LEN, num_words=NUM_WORDS, index_from=INDEX_FROM)
train_x,train_y = train
test_x,test_y = test

"""
Explanation: Load the IMDB dataset
End of explanation
"""


word_to_id = keras.datasets.imdb.get_word_index()
word_to_id = {k:(v+INDEX_FROM) for k,v in word_to_id.items()}
word_to_id["<PAD>"] = 0
word_to_id["<START>"] = 1
word_to_id["<UNK>"] = 2

id_to_word = {value:key for key,value in word_to_id.items()}
print(' '.join(id_to_word[id] for id in train_x[0] ))

"""
Explanation: Look at an example review
(Punctuation and capitalization are stripped)
End of explanation
"""


def get_input_fn(x_in, y_in, shuffle=True, epochs=1):
    def input_fn():
        ys = tf.contrib.data.Dataset.from_tensor_slices(y_in)
        
        # Convert x_in to a sparse tensor
        nested_sparse = [
            (np.array([[n]*len(x),range(len(x))]).T,x)
            for n,x in enumerate(x_in)
        ]
        
        indices = np.concatenate([idx for idx,value in nested_sparse], axis = 0)
        values = np.concatenate([value for idx,value in nested_sparse], axis = 0)
        
        max_len = max(len(ex) for ex in x_in)
        xs = tf.SparseTensor(indices = indices, values = values, dense_shape=[25000, max_len])
        
        xs = tf.contrib.data.Dataset.from_sparse_tensor_slices(xs)
        
        xs = xs.map(lambda *x: tf.sparse_tensor_to_dense(tf.SparseTensor(*x)))
        
        ds = tf.contrib.data.Dataset.zip([xs,ys]).repeat(epochs)
        
        if shuffle:
            ds = ds.shuffle(10000)
            
        ds = ds.batch(32)

        x,y = ds.make_one_shot_iterator().get_next()

        return {'word_ids':x},y
        
    return input_fn
    

"""
Explanation: Create the Input Function
End of explanation
"""


in_fn = get_input_fn(x_in = np.array([[1,1,1],[2,2],[3,3,3],[4,4,4,4],[5],[6,6],[7,7,7,7,7]]), 
                     y_in = np.array([1,2,3,4,5,6,7]))

x,y = in_fn()

init = tf.global_variables_initializer()

with tf.train.MonitoredSession() as sess:
    sess.run(init)
    x,y = sess.run([x,y])

"""
Explanation: Test the input function
End of explanation
"""


x['word_ids']

y[:,None]

"""
Explanation: Note the zero padding, and that x and y have the same shuffle applied.
End of explanation
"""


word_ids = tf.feature_column.categorical_column_with_identity('word_ids', NUM_WORDS)

bow_estimator = tf.contrib.learn.LinearClassifier(feature_columns=[word_ids], model_dir='tensorboard/BOW')

for n in range(25):
    bow_estimator.fit(input_fn=get_input_fn(train_x,train_y,epochs=10))
    bow_estimator.evaluate(input_fn=get_input_fn(test_x, test_y,epochs=1))

"""
Explanation: Build the Estimator
End of explanation
"""


tf.reset_default_graph()

DNN_bow_estimator = tf.contrib.learn.DNNClassifier(
    [256, 256],  model_dir='tensorboard/DNN_BOW',
    feature_columns=[tf.feature_column.embedding_column(word_ids, 30, combiner='mean')])

for n in range(25):
    DNN_bow_estimator.fit(input_fn=get_input_fn(train_x,train_y,epochs=10))
    DNN_bow_estimator.evaluate(input_fn=get_input_fn(test_x,test_y,epochs=1))

"""
Explanation: DNN Bag of words?
End of explanation
"""


