'Yield (token_type, str_data) tokens.
The last token will be (EOF, None) where EOF is the singleton object
defined in this module.'

def lex(self, text):

    for match in self.regex.finditer(text):
        for (name, _) in self.lexicon:
            m = match.group(name)
            if (m is not None):
                (yield (name, m))
                break
    (yield (EOF, None))
