code: def generate_tokens(readline):
      """
      The generate_tokens() generator requires one argment, readline, which
      must be a callable object which provides the same interface as the
      readline() method of built-in file objects. Each call to the function
      should return one line of input as a string.  Alternately, readline
      can be a callable function terminating with StopIteration:
          readline = open(myfile).next    # Example of alternate readline
  
      The generator produces 5-tuples with these members: the token type; the
      token string; a 2-tuple (srow, scol) of ints specifying the row and
      column where the token begins in the source; a 2-tuple (erow, ecol) of
      ints specifying the row and column where the token ends in the source;
      and the line on which the token was found. The line passed is the
      logical line; continuation lines are included.
      """
      lnum = parenlev = continued = 0
      namechars, numchars = string.ascii_letters + "_", "0123456789"
      contstr, needcont = "", 0
      contline = None
      indents = [0]
  
      while 1:  # loop over lines in stream
          try:
              line = readline()
          except StopIteration:
              line = ""
          lnum += 1
          pos, max = 0, len(line)
  
          if contstr:  # continued string
              if not line:
                  raise TokenError("EOF in multi-line string", strstart)
              endmatch = endprog.match(line)
              if endmatch:
                  pos = end = endmatch.end(0)
                  yield (
                      STRING,
                      contstr + line[:end],
                      strstart,
                      (lnum, end),
                      contline + line,
                  )
                  contstr, needcont = "", 0
                  contline = None
              elif (
                  needcont
                  and line[-2:] != "\\\n"
                  and line[-3:] != "\\\r\n"
              ):
                  yield (
                      ERRORTOKEN,
                      contstr + line,
                      strstart,
                      (lnum, len(line)),
                      contline,
                  )
                  contstr = ""
                  contline = None
                  continue
              else:
                  contstr = contstr + line
                  contline = contline + line
                  continue
  
          elif parenlev == 0 and not continued:  # new statement
              if not line:
                  break
              column = 0
              while pos < max:  # measure leading whitespace
                  if line[pos] == " ":
                      column += 1
                  elif line[pos] == "\t":
                      column = (column // tabsize + 1) * tabsize
                  elif line[pos] == "\f":
                      column = 0
                  else:
                      break
                  pos += 1
              if pos == max:
                  break
  
              if line[pos] in "#\r\n":  # skip comments or blank lines
                  if line[pos] == "#":
                      comment_token = line[pos:].rstrip("\r\n")
                      nl_pos = pos + len(comment_token)
                      yield (
                          COMMENT,
                          comment_token,
                          (lnum, pos),
                          (lnum, pos + len(comment_token)),
                          line,
                      )
                      yield (
                          NEWLINE,
                          line[nl_pos:],
                          (lnum, nl_pos),
                          (lnum, len(line)),
                          line,
                      )
                  else:
                      yield (
                          NEWLINE,
                          line[pos:],
                          (lnum, pos),
                          (lnum, len(line)),
                          line,
                      )
                  continue
  
              if column > indents[-1]:  # count indents or dedents
                  indents.append(column)
                  yield (
                      INDENT,
                      line[:pos],
                      (lnum, 0),
                      (lnum, pos),
                      line,
                  )
              while column < indents[-1]:
                  if column not in indents:
                      raise IndentationError(
                          "unindent does not match any outer indentation level",
                          ("<tokenize>", lnum, pos, line),
                      )
                  indents = indents[:-1]
                  yield (DEDENT, "", (lnum, pos), (lnum, pos), line)
  
          else:  # continued statement
              if not line:
                  raise TokenError(
                      "EOF in multi-line statement", (lnum, 0)
                  )
              continued = 0
  
          while pos < max:
              pseudomatch = pseudoprog.match(line, pos)
              if pseudomatch:  # scan for tokens
                  start, end = pseudomatch.span(1)
                  spos, epos, pos = (lnum, start), (lnum, end), end
                  token, initial = line[start:end], line[start]
  
                  if initial in numchars or (
                      initial == "." and token != "."
                  ):  # ordinary number
                      yield (NUMBER, token, spos, epos, line)
                  elif initial in "\r\n":
                      yield (
                          NL if parenlev > 0 else NEWLINE,
                          token,
                          spos,
                          epos,
                          line,
                      )
                  elif initial == "#":
                      assert not token.endswith("\n")
                      yield (COMMENT, token, spos, epos, line)
                  elif token in triple_quoted:
                      endprog = endprogs[token]
                      endmatch = endprog.match(line, pos)
                      if endmatch:  # all on one line
                          pos = endmatch.end(0)
                          token = line[start:pos]
                          yield (STRING, token, spos, (lnum, pos), line)
                      else:
                          strstart = (lnum, start)  # multiple lines
                          contstr = line[start:]
                          contline = line
                          break
                  elif (
                      initial in single_quoted
                      or token[:2] in single_quoted
                      or token[:3] in single_quoted
                  ):
                      if token[-1] == "\n":  # continued string
                          strstart = (lnum, start)
                          endprog = (
                              endprogs[initial]
                              or endprogs[token[1]]
                              or endprogs[token[2]]
                          )
                          contstr, needcont = line[start:], 1
                          contline = line
                          break
                      else:  # ordinary string
                          yield (STRING, token, spos, epos, line)
                  elif initial in namechars:  # ordinary name
                      yield (NAME, token, spos, epos, line)
                  elif initial == "\\":  # continued stmt
                      continued = 1
                  else:
                      if initial in "([{":
                          parenlev += 1
                      elif initial in ")]}":
                          parenlev -= 1
                      yield (OP, token, spos, epos, line)
              else:
                  yield (
                      ERRORTOKEN,
                      line[pos],
                      (lnum, pos),
                      (lnum, pos + 1),
                      line,
                  )
                  pos += 1
  
      for indent in indents[1:]:  # pop remaining indent levels
          yield (DEDENT, "", (lnum, 0), (lnum, 0), "")
      yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "")
  
imports: from __future__ import print_function
  import string, re
  from token import *
  import token
label_desc: Use image node_id 8 for calling a global function with example usage: generate_tokens(readline) without return types