'Loops over the styles in the document and stores important info.
When enough info is gathered, will perform a call to analyze the code
and generate subsequent language structures. These language structures
will later be used to generate XML output for the document.'

def token_next(self, style, text, start_column, start_line, **other_args):

    if (self.state == S_GET_HEREDOC_MARKER):
        if (not text.strip()):
            log.debug('Ignoring    whitespace    after    <<<:    %r', text)
            return
        self.heredocMarker = self._unquoteString(text)
        log.debug('getting    heredoc    marker:    %r,    now    in    heredoc    state', text)
        self._resetState(S_IN_HEREDOC)
    elif (self.state == S_IN_HEREDOC):
        if ((text == self.heredocMarker) and self.lastText and (self.lastText[(-1)] in '\r\n')):
            log.debug('end    of    heredoc:    %r', self.heredocMarker)
            self._resetState(self.return_to_state)
        else:
            log.debug('ignoring    heredoc    material')
    elif ((style in (self.PHP_WORD, self.PHP_IDENTIFIER, self.PHP_OPERATOR, self.PHP_NUMBER, self.PHP_VARIABLE)) or (style in self.PHP_STRINGS)):
        self.lineno = (start_line + 1)
        if (style != self.PHP_OPERATOR):
            if (style == self.PHP_IDENTIFIER):
                text = text.strip()
            if text:
                self.text.append(text)
                self.styles.append(style)
                self.linenos.append(self.lineno)
        else:
            if (text == '<<<'):
                self.return_to_state = self.state
                self.state = S_GET_HEREDOC_MARKER
            elif text.startswith('<?'):
                if (text[:5].lower() == '<?php'):
                    text = text[5:]
                elif text.startswith('<?='):
                    text = text[len('<?='):]
                else:
                    text = text[len('<?'):]
            elif text.startswith('<%'):
                if text.startswith('<%='):
                    text = text[len('<%='):]
                else:
                    text = text[len('<%'):]
            if text.endswith('?>'):
                text = text[:(- len('?>'))]
            elif text.endswith('<%'):
                text = text[:(- len('%>'))]
            col = (start_column + 1)
            for op in text:
                self.styles.append(style)
                self.text.append(op)
                self.linenos.append(self.lineno)
                if (op == '('):
                    self.return_to_state = self.state
                    self.state = S_IN_ARGS
                elif (op == ')'):
                    self.state = self.return_to_state
                elif (op == '='):
                    if (text == op):
                        self.state = S_IN_ASSIGNMENT
                elif (op == '{'):
                    self._addCodePiece()
                    self.incBlock()
                elif (op == '}'):
                    if (len(self.text) == 1):
                        self._resetState()
                    else:
                        self._addCodePiece()
                    self.decBlock()
                elif (op == ':'):
                    if ((len(self.text) > 0) and (self.styles[0] == self.PHP_WORD) and (self.text[0].lower() in ('if', 'elseif', 'else', 'while', 'for', 'foreach', 'switch'))):
                        self._addCodePiece()
                    elif (('case' in self.text) or ('default' in self.text)):
                        self._addCodePiece()
                elif (op == ';'):
                    if ((len(self.text) > 0) and (self.styles[0] == self.PHP_WORD) and (self.text[(-1)].lower() in ('endif', 'endwhile', 'endfor', 'endforeach', 'endswitch'))):
                        self.text = self.text[:(-1)]
                    self._addCodePiece()
                col += 1
    elif (style in self.PHP_COMMENT_STYLES):
        comment = text.rstrip()
        if (not self.text):
            if ((style == SCE_UDL_SSL_COMMENT) and self.comment and (start_line <= (self.comments[(-1)][2] + 1)) and (style == self.comments[(-1)][1])):
                self.comment += comment
            else:
                self.comment = comment
        self.comments.append([comment, style, start_line, start_column])
    elif ((style == SCE_UDL_SSL_DEFAULT) and (self.lastStyle in self.PHP_COMMENT_STYLES) and (text[0] in '\r\n')):
        if self.comment:
            self.comment += '\n'
        self.comments[(-1)][0] += '\n'
    elif is_udl_csl_style(style):
        self.csl_tokens.append({'style': style, 'text': text, 'start_column': start_column, 'start_line': start_line})
    self.lastText = text
    self.lastStyle = style
