'Tokenizes a text file.'

def tokenize(self, path):

    assert os.path.exists(path)
    with open(path, 'r') as f:
        tokens = 0
        for line in f:
            words = (line.split() + ['<eos>'])
            tokens += len(words)
            for word in words:
                self.dictionary.add_word(word)
    with open(path, 'r') as f:
        ids = np.zeros((tokens,), dtype='int32')
        token = 0
        for line in f:
            words = (line.split() + ['<eos>'])
            for word in words:
                ids[token] = self.dictionary.word2idx[word]
                token += 1
    return mx.nd.array(ids, dtype='int32')
