'Given a sequence of tokens, generates lists of tokens, each list
corresponding to a sentence.'

def sentences_from_tokens(self, tokens):

    tokens = iter(self._annotate_tokens((self._Token(t) for t in tokens)))
    sentence = []
    for aug_tok in tokens:
        sentence.append(aug_tok.tok)
        if aug_tok.sentbreak:
            (yield sentence)
            sentence = []
    if sentence:
        (yield sentence)
