from keras.datasets import imdb
idx = imdb.get_word_index()

"""
Explanation: Setup data
We're going to look at the IMDB dataset, which contains movie reviews from IMDB, along with their sentiment. Keras comes with some helpers for this dataset.
End of explanation
"""


idx_arr = sorted(idx, key=idx.get)
# sorted() is a built-in function: sorted()
# sorted(iterable, *, key=None, reverse=False)
# Return a new sorted list from the items in iterable.
idx_arr[:10]

"""
Explanation: This is the word list:
End of explanation
"""


idx2word = {v: k for k, v in idx.iteritems()}

"""
Explanation: ...and this is the mapping from id to word
End of explanation
"""


path = get_file('imdb_full.pkl',
                origin='https://s3.amazonaws.com/text-datasets/imdb_full.pkl',
                md5_hash='d091312047c43cf9e4e38fef92437263')
f = open(path, 'rb')
(x_train, labels_train), (x_test, labels_test) = pickle.load(f)

len(x_train)

"""
Explanation: We download the reviews using code copied from keras.datasets:
End of explanation
"""


', '.join(map(str, x_train[0]))

"""
Explanation: Here's the 1st review. As you see, the words have been replaced by ids. The ids can be looked up in idx2word.
End of explanation
"""


idx2word[23022]

"""
Explanation: The first word of the first review is 23022. Let's see what that is.
End of explanation
"""


' '.join([idx2word[o] for o in x_train[0]])

"""
Explanation: Here's the whole review, mapped from ids to words.
End of explanation
"""


labels_train[:10]

"""
Explanation: The labels are 1 for positive, 0 for negative.
End of explanation
"""


vocab_size = 5000

trn = [np.array([i if i<vocab_size-1 else vocab_size-1 for i in s]) for s in x_train]
test = [np.array([i if i<vocab_size-1 else vocab_size-1 for i in s]) for s in x_test]

"""
Explanation: Reduce vocab size by setting rare words to max index.
End of explanation
"""


lens = np.array(map(len, trn))
(lens.max(), lens.min(), lens.mean())

"""
Explanation: Look at distribution of lengths of sentences.
End of explanation
"""


seq_len = 500

trn = sequence.pad_sequences(trn, maxlen=seq_len, value=0)
test = sequence.pad_sequences(test, maxlen=seq_len, value=0)

"""
Explanation: Pad (with zero) or truncate each sentence to make consistent length.
End of explanation
"""


trn.shape

"""
Explanation: This results in nice rectangular matrices that can be passed to ML algorithms. Reviews shorter than 500 words are pre-padded with zeros, those greater are truncated.
End of explanation
"""


model = Sequential([
    Embedding(vocab_size, 32, input_length=seq_len),
    Flatten(),
    Dense(100, activation='relu'),
    Dropout(0.7),
    Dense(1, activation='sigmoid')])

model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])
model.summary()

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=2, batch_size=64)

"""
Explanation: Create simple models
Single hidden layer NN
The simplest model that tends to give reasonable results is a single hidden layer net. So let's try that. Note that we can't expect to get any useful results by feeding word ids directly into a neural net - so instead we use an embedding to replace them with a vector of 32 (initially random) floats for each word in the vocab.
End of explanation
"""


conv1 = Sequential([
    Embedding(vocab_size, 32, input_length=seq_len, dropout=0.2),
    Dropout(0.2),
    Convolution1D(64, 5, border_mode='same', activation='relu'),
    Dropout(0.2),
    MaxPooling1D(),
    Flatten(),
    Dense(100, activation='relu'),
    Dropout(0.7),
    Dense(1, activation='sigmoid')])

conv1.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])

conv1.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=4, batch_size=64)

"""
Explanation: The stanford paper that this dataset is from cites a state of the art accuracy (without unlabelled data) of 0.883. So we're short of that, but on the right track.
Single conv layer with max pooling
A CNN is likely to work better, since it's designed to take advantage of ordered data. We'll need to use a 1D CNN, since a sequence of words is 1D.
End of explanation
"""


conv1.save_weights(model_path + 'conv1.h5')

conv1.load_weights(model_path + 'conv1.h5')

"""
Explanation: That's well past the Stanford paper's accuracy - another win for CNNs!
End of explanation
"""


def get_glove_dataset(dataset):
    """Download the requested glove dataset from files.fast.ai
    and return a location that can be passed to load_vectors.
    """
    # see wordvectors.ipynb for info on how these files were
    # generated from the original glove data.
    md5sums = {'6B.50d': '8e1557d1228decbda7db6dfd81cd9909',
               '6B.100d': 'c92dbbeacde2b0384a43014885a60b2c',
               '6B.200d': 'af271b46c04b0b2e41a84d8cd806178d',
               '6B.300d': '30290210376887dcc6d0a5a6374d8255'}
    glove_path = os.path.abspath('data/glove/results')
    %mkdir -p $glove_path
    return get_file(dataset,
                    'http://files.fast.ai/models/glove/' + dataset + '.tgz',
                    cache_subdir=glove_path,
                    md5_hash=md5sums.get(dataset, None),
                    untar=True)

def load_vectors(loc):
    return (load_array(loc+'.dat'),
        pickle.load(open(loc+'_words.pkl','rb')),
        pickle.load(open(loc+'_idx.pkl','rb')))

vecs, words, wordidx = load_vectors(get_glove_dataset('6B.50d'))

"""
Explanation: Pre-trained vectors
You may want to look at wordvectors.ipynb before moving on.
In this section, we replicate the previous CNN, but using pre-trained embeddings.
End of explanation
"""


def create_emb():
    n_fact = vecs.shape[1]
    emb = np.zeros((vocab_size, n_fact))

    for i in range(1,len(emb)):
        word = idx2word[i]
        if word and re.match(r"^[a-zA-Z0-9\-]*$", word):
            src_idx = wordidx[word]
            emb[i] = vecs[src_idx]
        else:
            # If we can't find the word in glove, randomly initialize
            emb[i] = normal(scale=0.6, size=(n_fact,))

    # This is our "rare word" id - we want to randomly initialize
    emb[-1] = normal(scale=0.6, size=(n_fact,))
    emb/=3
    return emb

emb = create_emb()

"""
Explanation: The glove word ids and imdb word ids use different indexes. So we create a simple function that creates an embedding matrix using the indexes from imdb, and the embeddings from glove (where they exist).
End of explanation
"""


model = Sequential([
    Embedding(vocab_size, 50, input_length=seq_len, dropout=0.2, 
              weights=[emb], trainable=False),
    Dropout(0.25),
    Convolution1D(64, 5, border_mode='same', activation='relu'),
    Dropout(0.25),
    MaxPooling1D(),
    Flatten(),
    Dense(100, activation='relu'),
    Dropout(0.7),
    Dense(1, activation='sigmoid')])

model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=2, batch_size=64)

"""
Explanation: We pass our embedding matrix to the Embedding constructor, and set it to non-trainable.
End of explanation
"""


model.layers[0].trainable=True

model.optimizer.lr=1e-4

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=1, batch_size=64)

"""
Explanation: We already have beaten our previous model! But let's fine-tune the embedding weights - especially since the words we couldn't find in glove just have random embeddings.
End of explanation
"""


model.save_weights(model_path+'glove50.h5')

"""
Explanation: As expected, that's given us a nice little boost. :)
End of explanation
"""


from keras.layers import Merge

"""
Explanation: Multi-size CNN
This is an implementation of a multi-size CNN as shown in Ben Bowles' excellent blog post.
End of explanation
"""


graph_in = Input ((vocab_size, 50))
convs = [ ] 
for fsz in range (3, 6): 
    x = Convolution1D(64, fsz, border_mode='same', activation="relu")(graph_in)
    x = MaxPooling1D()(x) 
    x = Flatten()(x) 
    convs.append(x)
out = Merge(mode="concat")(convs) 
graph = Model(graph_in, out) 

emb = create_emb()

"""
Explanation: We use the functional API to create multiple conv layers of different sizes, and then concatenate them.
End of explanation
"""


model = Sequential ([
    Embedding(vocab_size, 50, input_length=seq_len, dropout=0.2, weights=[emb]),
    Dropout (0.2),
    graph,
    Dropout (0.5),
    Dense (100, activation="relu"),
    Dropout (0.7),
    Dense (1, activation='sigmoid')
    ])

model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=2, batch_size=64)

"""
Explanation: We then replace the conv/max-pool layer in our original CNN with the concatenated conv layers.
End of explanation
"""


model.layers[0].trainable=False

model.optimizer.lr=1e-5

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=2, batch_size=64)

"""
Explanation: Interestingly, I found that in this case I got best results when I started the embedding layer as being trainable, and then set it to non-trainable after a couple of epochs. I have no idea why!
End of explanation
"""


model = Sequential([
    Embedding(vocab_size, 32, input_length=seq_len, mask_zero=True,
              W_regularizer=l2(1e-6), dropout=0.2),
    LSTM(100, consume_less='gpu'),
    Dense(1, activation='sigmoid')])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

model.fit(trn, labels_train, validation_data=(test, labels_test), nb_epoch=5, batch_size=64)

"""
Explanation: This more complex architecture has given us another boost in accuracy.
LSTM
We haven't covered this bit yet!
End of explanation
"""


