'Load vocab and initialize buffers
Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size)
where each column is the 1-hot representation of a word and the first batch_size columns
are the first words of each sentence.'

def load_vocab(self):

    sentences = [sent['tokens'] for sent in self.iterSentences()]
    words = [word for sentence in sentences for word in sentence]
    word_counts = Counter(words)
    vocab = ([self.end_token] + [word for word in list(word_counts.keys()) if (word_counts[word] >= 5)])
    self.vocab_size = len(vocab)
    self.vocab_to_index = dict(((c, i) for (i, c) in enumerate(vocab)))
    self.index_to_vocab = dict(((i, c) for (i, c) in enumerate(vocab)))
    word_counts[self.end_token] = len(sentences)
    self.bias_init = np.array([(1.0 * word_counts[self.index_to_vocab[i]]) for i in self.index_to_vocab]).reshape((self.vocab_size, 1))
    self.bias_init /= np.sum(self.bias_init)
    self.bias_init = np.log(self.bias_init)
    self.bias_init -= np.max(self.bias_init)
    self.max_sentence_length = (max((len(sent) for sent in sentences)) + 1)
    self.dev_image = self.be.iobuf(self.image_size)
    self.dev_imageT = self.be.empty(self.dev_image.shape[::(-1)])
    self.dev_X = self.be.iobuf((self.vocab_size, self.max_sentence_length))
    self.dev_y = self.be.iobuf((self.vocab_size, (self.max_sentence_length + 1)))
    self.dev_y_mask = self.be.iobuf((self.vocab_size, (self.max_sentence_length + 1)))
    self.y_mask = np.zeros(self.dev_y_mask.shape, dtype=np.uint8).reshape(self.vocab_size, (self.max_sentence_length + 1), (-1))
    self.y_mask_reshape = self.y_mask.reshape(self.dev_y_mask.shape)
    self.dev_lbl = self.be.iobuf(self.max_sentence_length, dtype=np.int32)
    self.dev_lblT = self.be.empty(self.dev_lbl.shape[::(-1)])
    self.dev_lblflat = self.dev_lbl.reshape((1, self.dev_lbl.size))
    self.dev_y_lbl = self.be.iobuf((self.max_sentence_length + 1), dtype=np.int32)
    self.dev_y_lblT = self.be.empty(self.dev_y_lbl.shape[::(-1)])
    self.dev_y_lblflat = self.dev_y_lbl.reshape((1, self.dev_y_lbl.size))
    self.shape = [self.image_size, (self.vocab_size, self.max_sentence_length)]
    neon_logger.display(('Vocab    size:    %d,    Max    sentence    length:    %d' % (self.vocab_size, self.max_sentence_length)))
