'Lexically analyze `text`.
Yield pairs (`name`, `tokentext`).'

def lex(self, text):

    end = len(text)
    state = self.state
    regexes = self.regexes
    toks = self.toks
    start = 0
    while (start < end):
        for match in regexes[state].finditer(text, start):
            name = match.lastgroup
            tok = toks[name]
            toktext = match.group(name)
            start += len(toktext)
            (yield (tok.name, toktext))
            if tok.next:
                state = tok.next
                break
    self.state = state
