'This is a Python port of the Penn treebank tokenizer adapted by the Moses
machine translation community. It\'s a little different from the
version in nltk.tokenize.treebank.'

def penn_tokenize(self, text, return_str=False):

    text = text_type(text)
    for (regexp, substitution) in self.MOSES_PENN_REGEXES_1:
        text = re.sub(regexp, substitution, text)
    text = self.handles_nonbreaking_prefixes(text)
    for (regexp, substitution) in self.MOSES_PENN_REGEXES_2:
        text = re.sub(regexp, substitution, text)
    return (text if return_str else text.split())
