'Returns a sequence of tokens from the iterable.'

def tokenize(self, text, building=False):

    return (token for sent in self._sent_tokenize(text, building) for token in self._word_tokenize(sent, building))
