'SilverCity doesn\'t know much about Perl, and breaks in two ways:
1. It doesn\'t know how to separate sequences of characters into
separate tokens.
2. It doesn\'t know how to map DATASECTIONs into POD sequences.
It\'s easier to do this once before processing tokens individually.
This should all be done in silvercity.  Doing this has leaked the
whole silvercity abstraction into this module, and it doesn\'t
belong here.  This routine works with SilverCity tokens, not
shared_lexer Tokens.'

def _fix_token_list(self, **tok):

    if (tok['start_column'] > shared_lexer.MAX_REASONABLE_LIMIT):
        return
    ttype = tok['style']
    tval = tok['text']
    if ((ttype in (ScintillaConstants.SCE_PL_OPERATOR, ScintillaConstants.SCE_PL_VARIABLE_INDEXER)) and (len(tok['text']) > 1)):
        self.append_split_tokens(tok, self.multi_char_ops)
    elif (ttype == ScintillaConstants.SCE_PL_IDENTIFIER):
        tok['text'] = tok['text'].strip()
        self.complete_token_push(tok)
    elif ((not self._provide_full_docs) and (ttype in (ScintillaConstants.SCE_PL_DATASECTION, ScintillaConstants.SCE_PL_POD))):
        pass
    elif (ttype == ScintillaConstants.SCE_PL_DATASECTION):
        if pod_markings.search(tval):
            col = tok['start_column']
            for new_vals in ((ScintillaConstants.SCE_PL_WORD, 'package'), (ScintillaConstants.SCE_PL_IDENTIFIER, 'main'), (ScintillaConstants.SCE_PL_OPERATOR, ';')):
                (new_type, new_text) = new_vals
                new_tok = copy.copy(tok)
                new_tok['text'] = new_text
                new_tok['style'] = new_type
                new_tok['start_column'] = col
                new_tok['end_column'] = ((col + len(new_text)) - 1)
                col = (new_tok['end_column'] + 1)
                self.complete_token_push(new_tok)
            tok['style'] = ScintillaConstants.SCE_PL_POD
            tok['text'] = tval
            tok['start_column'] = col
            if (tok['start_line'] == tok['end_line']):
                tok['end_column'] = ((tok['start_line'] + len(tok['text'])) - 1)
            self.complete_token_push(tok)
        else:
            pass
    else:
        self.complete_token_push(tok)
