'Uses nltk Treebank Word Tokenizer for tokenizing words within
sentences.'

def _word_tokenize(self, text, building=False):

    word_tokens = self.word_tok.tokenize(text)
    if ((not building) and (self.max_ngram_size > 1)):
        word_tokens = find_ngrams(self.tok2ind, word_tokens, self.max_ngram_size)
    return word_tokens
