def source_token_lines(source):
ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])<EOL>line = []<EOL>col = <NUM_LIT:0><EOL>source = source.expandtabs(<NUM_LIT:8>).replace('<STR_LIT:\r\n>', '<STR_LIT:\n>')<EOL>tokgen = generate_tokens(source)<EOL>for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):<EOL><INDENT>mark_start = True<EOL>for part in re.split('<STR_LIT>', ttext):<EOL><INDENT>if part == '<STR_LIT:\n>':<EOL><INDENT>yield line<EOL>line = []<EOL>col = <NUM_LIT:0><EOL>mark_end = False<EOL><DEDENT>elif part == '<STR_LIT>':<EOL><INDENT>mark_end = False<EOL><DEDENT>elif ttype in ws_tokens:<EOL><INDENT>mark_end = False<EOL><DEDENT>else:<EOL><INDENT>if mark_start and scol > col:<EOL><INDENT>line.append(("<STR_LIT>", "<STR_LIT:U+0020>" * (scol - col)))<EOL>mark_start = False<EOL><DEDENT>tok_class = tokenize.tok_name.get(ttype, '<STR_LIT>').lower()[:<NUM_LIT:3>]<EOL>if ttype == token.NAME and keyword.iskeyword(ttext):<EOL><INDENT>tok_class = "<STR_LIT:key>"<EOL><DEDENT>line.append((tok_class, part))<EOL>mark_end = True<EOL><DEDENT>scol = <NUM_LIT:0><EOL><DEDENT>if mark_end:<EOL><INDENT>col = ecol<EOL><DEDENT><DEDENT>if line:<EOL><INDENT>yield line<EOL><DEDENT>
Generate a series of lines, one for each line in `source`.

    Each line is a list of pairs, each pair is a token::

        [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]

    Each pair has a token class, and the token text.

    If you concatenate all the token texts, and then join them with newlines,
    you should have your original `source` back, with two differences:
    trailing whitespace is not preserved, and a final line with no newline
    is indistinguishable from a final line with a newline.