'Split ``text`` into (tokentype, text) pairs.
If ``context`` is given, use this lexer context instead.'

def get_tokens_unprocessed(self, text=None, context=None):

    tokendefs = self._tokens
    if (not context):
        ctx = LexerContext(text, 0)
        statetokens = tokendefs['root']
    else:
        ctx = context
        statetokens = tokendefs[ctx.stack[(-1)]]
        text = ctx.text
    while 1:
        for (rexmatch, action, new_state) in statetokens:
            m = rexmatch(text, ctx.pos, ctx.end)
            if m:
                if (type(action) is _TokenType):
                    (yield (ctx.pos, action, m.group()))
                    ctx.pos = m.end()
                else:
                    for item in action(self, m, ctx):
                        (yield item)
                    if (not new_state):
                        statetokens = tokendefs[ctx.stack[(-1)]]
                if (new_state is not None):
                    if isinstance(new_state, tuple):
                        ctx.stack.extend(new_state)
                    elif isinstance(new_state, int):
                        del ctx.stack[new_state:]
                    elif (new_state == '#push'):
                        ctx.stack.append(ctx.stack[(-1)])
                    else:
                        assert False, ('wrong    state    def:    %r' % new_state)
                    statetokens = tokendefs[ctx.stack[(-1)]]
                break
        else:
            try:
                if (ctx.pos >= ctx.end):
                    break
                if (text[ctx.pos] == '\n'):
                    ctx.pos += 1
                    ctx.stack = ['root']
                    statetokens = tokendefs['root']
                    (yield (ctx.pos, Text, u'\n'))
                    continue
                (yield (ctx.pos, Error, text[ctx.pos]))
                ctx.pos += 1
            except IndexError:
                break
