'Divide the given text into tokens, using the punkt word
segmentation regular expression, and generate the resulting list
of tokens augmented as three-tuples with two boolean values for whether
the given token occurs at the start of a paragraph or a new line,
respectively.'

def _tokenize_words(self, plaintext):

    parastart = False
    for line in plaintext.split(u'\n'):
        if line.strip():
            line_toks = iter(self._lang_vars.word_tokenize(line))
            (yield self._Token(next(line_toks), parastart=parastart, linestart=True))
            parastart = False
            for t in line_toks:
                (yield self._Token(t))
        else:
            parastart = True
