code: def generate_tokens(
      readline: Callable[[], str], grammar: Optional[Grammar] = None
  ) -> Iterator[GoodTokenInfo]:
      """
      The generate_tokens() generator requires one argument, readline, which
      must be a callable object which provides the same interface as the
      readline() method of built-in file objects. Each call to the function
      should return one line of input as a string.  Alternately, readline
      can be a callable function terminating with StopIteration:
          readline = open(myfile).next    # Example of alternate readline
  
      The generator produces 5-tuples with these members: the token type; the
      token string; a 2-tuple (srow, scol) of ints specifying the row and
      column where the token begins in the source; a 2-tuple (erow, ecol) of
      ints specifying the row and column where the token ends in the source;
      and the line on which the token was found. The line passed is the
      logical line; continuation lines are included.
      """
      lnum = parenlev = continued = 0
      numchars: Final[str] = "0123456789"
      contstr, needcont = "", 0
      contline: Optional[str] = None
      indents = [0]
  
      # If we know we're parsing 3.7+, we can unconditionally parse `async` and
      # `await` as keywords.
      async_keywords = (
          False if grammar is None else grammar.async_keywords
      )
      # 'stashed' and 'async_*' are used for async/await parsing
      stashed: Optional[GoodTokenInfo] = None
      async_def = False
      async_def_indent = 0
      async_def_nl = False
  
      strstart: Tuple[int, int]
      endprog: Pattern[str]
  
      while 1:  # loop over lines in stream
          try:
              line = readline()
          except StopIteration:
              line = ""
          lnum += 1
          pos, max = 0, len(line)
  
          if contstr:  # continued string
              assert contline is not None
              if not line:
                  raise TokenError("EOF in multi-line string", strstart)
              endmatch = endprog.match(line)
              if endmatch:
                  pos = end = endmatch.end(0)
                  yield (
                      STRING,
                      contstr + line[:end],
                      strstart,
                      (lnum, end),
                      contline + line,
                  )
                  contstr, needcont = "", 0
                  contline = None
              elif (
                  needcont
                  and line[-2:] != "\\\n"
                  and line[-3:] != "\\\r\n"
              ):
                  yield (
                      ERRORTOKEN,
                      contstr + line,
                      strstart,
                      (lnum, len(line)),
                      contline,
                  )
                  contstr = ""
                  contline = None
                  continue
              else:
                  contstr = contstr + line
                  contline = contline + line
                  continue
  
          elif parenlev == 0 and not continued:  # new statement
              if not line:
                  break
              column = 0
              while pos < max:  # measure leading whitespace
                  if line[pos] == " ":
                      column += 1
                  elif line[pos] == "\t":
                      column = (column // tabsize + 1) * tabsize
                  elif line[pos] == "\f":
                      column = 0
                  else:
                      break
                  pos += 1
              if pos == max:
                  break
  
              if stashed:
                  yield stashed
                  stashed = None
  
              if line[pos] in "\r\n":  # skip blank lines
                  yield (
                      NL,
                      line[pos:],
                      (lnum, pos),
                      (lnum, len(line)),
                      line,
                  )
                  continue
  
              if line[pos] == "#":  # skip comments
                  comment_token = line[pos:].rstrip("\r\n")
                  nl_pos = pos + len(comment_token)
                  yield (
                      COMMENT,
                      comment_token,
                      (lnum, pos),
                      (lnum, nl_pos),
                      line,
                  )
                  yield (
                      NL,
                      line[nl_pos:],
                      (lnum, nl_pos),
                      (lnum, len(line)),
                      line,
                  )
                  continue
  
              if column > indents[-1]:  # count indents
                  indents.append(column)
                  yield (
                      INDENT,
                      line[:pos],
                      (lnum, 0),
                      (lnum, pos),
                      line,
                  )
  
              while column < indents[-1]:  # count dedents
                  if column not in indents:
                      raise IndentationError(
                          "unindent does not match any outer indentation level",
                          ("<tokenize>", lnum, pos, line),
                      )
                  indents = indents[:-1]
  
                  if async_def and async_def_indent >= indents[-1]:
                      async_def = False
                      async_def_nl = False
                      async_def_indent = 0
  
                  yield (DEDENT, "", (lnum, pos), (lnum, pos), line)
  
              if (
                  async_def
                  and async_def_nl
                  and async_def_indent >= indents[-1]
              ):
                  async_def = False
                  async_def_nl = False
                  async_def_indent = 0
  
          else:  # continued statement
              if not line:
                  raise TokenError(
                      "EOF in multi-line statement", (lnum, 0)
                  )
              continued = 0
  
          while pos < max:
              pseudomatch = pseudoprog.match(line, pos)
              if pseudomatch:  # scan for tokens
                  start, end = pseudomatch.span(1)
                  spos, epos, pos = (lnum, start), (lnum, end), end
                  token, initial = line[start:end], line[start]
  
                  if initial in numchars or (
                      initial == "." and token != "."
                  ):  # ordinary number
                      yield (NUMBER, token, spos, epos, line)
                  elif initial in "\r\n":
                      newline = NEWLINE
                      if parenlev > 0:
                          newline = NL
                      elif async_def:
                          async_def_nl = True
                      if stashed:
                          yield stashed
                          stashed = None
                      yield (newline, token, spos, epos, line)
  
                  elif initial == "#":
                      assert not token.endswith("\n")
                      if stashed:
                          yield stashed
                          stashed = None
                      yield (COMMENT, token, spos, epos, line)
                  elif token in triple_quoted:
                      endprog = endprogs[token]
                      endmatch = endprog.match(line, pos)
                      if endmatch:  # all on one line
                          pos = endmatch.end(0)
                          token = line[start:pos]
                          if stashed:
                              yield stashed
                              stashed = None
                          yield (STRING, token, spos, (lnum, pos), line)
                      else:
                          strstart = (lnum, start)  # multiple lines
                          contstr = line[start:]
                          contline = line
                          break
                  elif (
                      initial in single_quoted
                      or token[:2] in single_quoted
                      or token[:3] in single_quoted
                  ):
                      if token[-1] == "\n":  # continued string
                          strstart = (lnum, start)
                          maybe_endprog = (
                              endprogs.get(initial)
                              or endprogs.get(token[1])
                              or endprogs.get(token[2])
                          )
                          assert (
                              maybe_endprog is not None
                          ), f"endprog not found for {token}"
                          endprog = maybe_endprog
                          contstr, needcont = line[start:], 1
                          contline = line
                          break
                      else:  # ordinary string
                          if stashed:
                              yield stashed
                              stashed = None
                          yield (STRING, token, spos, epos, line)
                  elif initial.isidentifier():  # ordinary name
                      if token in ("async", "await"):
                          if async_keywords or async_def:
                              yield (
                                  ASYNC if token == "async" else AWAIT,
                                  token,
                                  spos,
                                  epos,
                                  line,
                              )
                              continue
  
                      tok = (NAME, token, spos, epos, line)
                      if token == "async" and not stashed:
                          stashed = tok
                          continue
  
                      if token in ("def", "for"):
                          if (
                              stashed
                              and stashed[0] == NAME
                              and stashed[1] == "async"
                          ):
                              if token == "def":
                                  async_def = True
                                  async_def_indent = indents[-1]
  
                              yield (
                                  ASYNC,
                                  stashed[1],
                                  stashed[2],
                                  stashed[3],
                                  stashed[4],
                              )
                              stashed = None
  
                      if stashed:
                          yield stashed
                          stashed = None
  
                      yield tok
                  elif initial == "\\":  # continued stmt
                      # This yield is new; needed for better idempotency:
                      if stashed:
                          yield stashed
                          stashed = None
                      yield (NL, token, spos, (lnum, pos), line)
                      continued = 1
                  else:
                      if initial in "([{":
                          parenlev += 1
                      elif initial in ")]}":
                          parenlev -= 1
                      if stashed:
                          yield stashed
                          stashed = None
                      yield (OP, token, spos, epos, line)
              else:
                  yield (
                      ERRORTOKEN,
                      line[pos],
                      (lnum, pos),
                      (lnum, pos + 1),
                      line,
                  )
                  pos += 1
  
      if stashed:
          yield stashed
          stashed = None
  
      for _indent in indents[1:]:  # pop remaining indent levels
          yield (DEDENT, "", (lnum, 0), (lnum, 0), "")
      yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "")
  
imports: import sys
  from typing import Callable, Final, Iterable, Iterator, List, Optional, Pattern, Set, Tuple, Union
  from blib2to3.pgen2.grammar import Grammar
  from blib2to3.pgen2.token import ASYNC, AWAIT, COMMENT, DEDENT, ENDMARKER, ERRORTOKEN, INDENT, NAME, NEWLINE, NL, NUMBER, OP, STRING, tok_name
  import re
  from codecs import BOM_UTF8, lookup
  from .None import token
label_desc: Use image node_id 11 for calling a global function with example usage: generate_tokens(readline, grammar) without return types