code: def python_tokenize(code):
      # Since formulas can only contain Python expressions, and Python
      # expressions cannot meaningfully contain newlines, we'll just remove all
      # the newlines up front to avoid any complications:
      code = code.replace("\n", " ").strip()
      it = tokenize.generate_tokens(StringIO(code).readline)
      try:
          for pytype, string, (_, start), (_, end), code in it:
              if pytype == tokenize.ENDMARKER:
                  break
              if pytype in (tokenize.NL, tokenize.NEWLINE):
                  assert string == ""
                  continue
              origin = Origin(code, start, end)
              if pytype == tokenize.ERRORTOKEN:
                  raise PatsyError(
                      "error tokenizing input "
                      "(maybe an unclosed string?)",
                      origin,
                  )
              if pytype == tokenize.COMMENT:
                  raise PatsyError("comments are not allowed", origin)
              yield (pytype, string, origin)
          else:  # pragma: no cover
              raise ValueError("stream ended without ENDMARKER?!?")
      except tokenize.TokenError as e:
          # TokenError is raised iff the tokenizer thinks that there is
          # some sort of multi-line construct in progress (e.g., an
          # unclosed parentheses, which in Python lets a virtual line
          # continue past the end of the physical line), and it hits the
          # end of the source text. We have our own error handling for
          # such cases, so just treat this as an end-of-stream.
          #
          if "unterminated string literal" in e.args[0]:
              raise PatsyError(
                  "error tokenizing input ({})".format(e.args[0]),
                  Origin(code, 0, len(code)),
              )
  
          # Just in case someone adds some other error case:
          assert "EOF in multi-line" in e.args[0]
          return
  
file_path: patsy/patsy/tokens.py
imports: import tokenize
  from six.moves import cStringIO
  from patsy import PatsyError
  from patsy.origin import Origin
label_desc: Use image node_id 1 for calling a global function with example usage: python_tokenize(code) without return types