environment/lib/python2.7/site-packages/IPython/frontend/qt/console/pygments_highlighter.py
def get_tokens_unprocessed(self, text, stack=('root',)):
    """ Split ``text`` into (tokentype, text) pairs.

        Monkeypatched to store the final stack on the object itself.
    """
    pos = 0
    tokendefs = self._tokens
    if hasattr(self, '_saved_state_stack'):
        statestack = list(self._saved_state_stack)
    else:
        statestack = list(stack)
    statetokens = tokendefs[statestack[-1]]
    while 1:
        for rexmatch, action, new_state in statetokens:
            m = rexmatch(text, pos)
            if m:
                if type(action) is _TokenType:
                    yield pos, action, m.group()
                else:
                    for item in action(self, m):
                        yield item
                pos = m.end()
                if new_state is not None:
                    # state transition
                    if isinstance(new_state, tuple):
                        for state in new_state:
                            if state == '#pop':
                                statestack.pop()
                            elif state == '#push':
                                statestack.append(statestack[-1])
                            else:
                                statestack.append(state)
                    elif isinstance(new_state, int):
                        # pop
                        del statestack[new_state:]
                    elif new_state == '#push':
                        statestack.append(statestack[-1])
                    else:
                        assert False, "wrong state def: %r" % new_state
                    statetokens = tokendefs[statestack[-1]]
                break
        else:
            try:
                if text[pos] == '\n':
                    # at EOL, reset state to "root"
                    pos += 1
                    statestack = ['root']
                    statetokens = tokendefs['root']
                    yield pos, Text, u'\n'
                    continue
                yield pos, Error, text[pos]
                pos += 1
            except IndexError:
                break
    self._saved_state_stack = list(statestack)
def get_tokens_unprocessed(self, text, stack=('root',)):
    """ Split ``text`` into (tokentype, text) pairs.

        Monkeypatched to store the final stack on the object itself.
    """
    pos = 0
    tokendefs = self._tokens
    if hasattr(self, '_saved_state_stack'):
        statestack = list(self._saved_state_stack)
    else:
        statestack = list(stack)
    statetokens = tokendefs[statestack[-1]]
    while 1:
        for rexmatch, action, new_state in statetokens:
            m = rexmatch(text, pos)
            if m:
                if type(action) is _TokenType:
                    yield pos, action, m.group()
                else:
                    for item in action(self, m):
                        yield item
                pos = m.end()
                if new_state is not None:
                    # state transition
                    if isinstance(new_state, tuple):
                        for state in new_state:
                            if state == '#pop':
                                statestack.pop()
                            elif state == '#push':
                                statestack.append(statestack[-1])
                            else:
                                statestack.append(state)
                    elif isinstance(new_state, int):
                        # pop
                        del statestack[new_state:]
                    elif new_state == '#push':
                        statestack.append(statestack[-1])
                    else:
                        assert False, "wrong state def: %r" % new_state
                    statetokens = tokendefs[statestack[-1]]
                break
        else:
            try:
                if text[pos] == '\n':
                    # at EOL, reset state to "root"
                    pos += 1
                    statestack = ['root']
                    statetokens = tokendefs['root']
                    yield pos, Text, u'\n'
                    continue
                yield pos, Error, text[pos]
                pos += 1
            except IndexError:
                break
    self._saved_state_stack = list(statestack)
def get_tokens_unprocessed ( self , text , stack = ( 'root' , ) ) : pos = 0 tokendefs = self . _tokens if hasattr ( self , '_saved_state_stack' ) : statestack = list ( self . _saved_state_stack ) else : statestack = list ( stack ) statetokens = tokendefs [ statestack [ - 1 ] ] while 1 : for rexmatch , action , new_state in statetokens : m = rexmatch ( text , pos ) if m : if type ( action ) is _TokenType : yield pos , action , m . group ( ) else : for item in action ( self , m ) : yield item pos = m . end ( ) if new_state is not None : # state transition if isinstance ( new_state , tuple ) : for state in new_state : if state == '#pop' : statestack . pop ( ) elif state == '#push' : statestack . append ( statestack [ - 1 ] ) else : statestack . append ( state ) elif isinstance ( new_state , int ) : # pop del statestack [ new_state : ] elif new_state == '#push' : statestack . append ( statestack [ - 1 ] ) else : assert False , "wrong state def: %r" % new_state statetokens = tokendefs [ statestack [ - 1 ] ] break else : try : if text [ pos ] == '\n' : # at EOL, reset state to "root" pos += 1 statestack = [ 'root' ] statetokens = tokendefs [ 'root' ] yield pos , Text , u'\n' continue yield pos , Error , text [ pos ] pos += 1 except IndexError : break self . _saved_state_stack = list ( statestack )
Split ``text`` into (tokentype, text) pairs.

        Monkeypatched to store the final stack on the object itself.
Split text into ( tokentype text ) pairs .