code: def test_tokenization():
      """
      Test function for ``pyparser._normalize_tokens()``.
  
      Even though this function is private, it is extremely important to verify that it behaves correctly. In
      particular, we want to check that it does not break the round-trip guarantee of the tokenizer, and that it
      fixes all the problems that the original tokenizer has.
      """
  
      # Helper functions
      def _parse_to_tokens(text):
          """Parse text into tokens and then normalize them."""
          gen = iter(text.splitlines(True))  # True = keep newlines
          readline = gen.next if hasattr(gen, "next") else gen.__next__
          return pyparser._tokenize(readline)
  
      def _unparse_tokens(tokens):
          """Convert tokens back into the source code."""
          return tokenize.untokenize(t.token for t in tokens)
  
      def _assert_tokens(tokens, target):
          """Check that the tokens list corresponds to the target provided."""
          for i in range(len(tokens)):
              assert i < len(target), "Token %d %r not expected" % (
                  i,
                  tokens[i],
              )
              tok = tokens[i]
              trg = target[i]
              valid = False
              if isinstance(trg, int):
                  if tok.op == trg:
                      valid = True
                  name = token_names[trg]
              elif isinstance(trg, tuple) and len(trg) == 2:
                  if tok.op == trg[0] and tok.str == trg[1]:
                      valid = True
                  name = "%s(%s)" % (token_names[trg[0]], trg[1])
              else:
                  assert False, "Unknown target: %r" % trg
              if not valid:
                  assert (
                      False
                  ), "Mismatched token %d: found %r, should be %r" % (
                      i,
                      tok,
                      name,
                  )
          assert len(target) == len(
              tokens
          ), "Expected too many tokens: %d vs %d" % (
              len(tokens),
              len(target),
          )
  
      def check_code(code, expected_tokens=None, filename=None):
          """Test parsing of the given piece of code."""
          code = textwrap.dedent(code)
          if filename:
              print("Testing tokenization of %s:" % filename, end=" ")
          else:
              check_code.index = getattr(check_code, "index", 0) + 1
              print(
                  "Testing tokenization %d:" % check_code.index, end=" "
              )
          tokens = _parse_to_tokens(code)
          try:
              try:
                  unparsed = _unparse_tokens(tokens)
              except ValueError as e:
                  raise AssertionError("Cannot unparse tokens: %s" % e)
              assert_same_code(code, unparsed)
              if expected_tokens:
                  _assert_tokens(tokens, expected_tokens)
              print("ok")
          except AssertionError as e:
              print("Error: %s" % e)
              print("Original code fragment:\n" + code)
              print("Tokens:")
              for i, tok in enumerate(tokens):
                  print("%3d %r" % (i, tok))
              raise
  
      check_code(
          """
          try:
              while True:
                  pass
                  # comment
          except: pass
          """,
          [
              NL,
              NAME("try"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("while"),
              NAME("True"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              COMMENT,
              NL,
              DEDENT,
              DEDENT,
              NAME("except"),
              OP(":"),
              NAME("pass"),
              NEWLINE,
              END,
          ],
      )
      check_code(
          """
          try:
              while True:
                  pass
              # comment
          except: pass
          """,
          [
              NL,
              NAME("try"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("while"),
              NAME("True"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              COMMENT,
              NL,
              DEDENT,
              NAME("except"),
              OP(":"),
              NAME("pass"),
              NEWLINE,
              END,
          ],
      )
      check_code(
          """
          try:
              while True:
                  pass
          # comment
          except: pass
          """,
          [
              NL,
              NAME("try"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("while"),
              NAME("True"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              DEDENT,
              COMMENT,
              NL,
              NAME("except"),
              OP(":"),
              NAME("pass"),
              NEWLINE,
              END,
          ],
      )
      check_code(
          """
          def func():
              # function
              pass
          """,
          [
              NL,
              NAME("def"),
              NAME("func"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              COMMENT,
              NL,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          def func():  # function
                       # hanging comment
              pass
          """,
          [
              NL,
              NAME("def"),
              NAME("func"),
              OP("("),
              OP(")"),
              OP(":"),
              COMMENT,
              NEWLINE,
              INDENT,
              COMMENT,
              NL,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          def foo():
              pass
  
          #comment
          def bar():
              pass
          """,
          [
              NL,
              NAME("def"),
              NAME("foo"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              NL,
              COMMENT,
              NL,
              NAME("def"),
              NAME("bar"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          def hello():
  
  
              print("hello")
          """,
          [
              NL,
              NAME("def"),
              NAME("hello"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NL,
              NL,
              NAME("print"),
              OP("("),
              STRING,
              OP(")"),
              NEWLINE,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          class Foo:
              def foo(self):
                  pass
  
              def bar(self):
                  return
          """,
          [
              NL,
              NAME("class"),
              NAME("Foo"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("def"),
              NAME("foo"),
              OP("("),
              NAME("self"),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              NL,
              NAME("def"),
              NAME("bar"),
              OP("("),
              NAME("self"),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("return"),
              NEWLINE,
              DEDENT,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          def foo():
              # Attempt to create the output directory
              try:
                  os.makedirs(destdir)
              except OSError as e:
                  raise
          """,
          [
              NL,
              NAME("def"),
              NAME("foo"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              COMMENT,
              NL,
              NAME("try"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("os"),
              OP("."),
              NAME("makedirs"),
              OP("("),
              NAME("destdir"),
              OP(")"),
              NEWLINE,
              DEDENT,
              NAME("except"),
              NAME("OSError"),
              NAME("as"),
              NAME("e"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("raise"),
              NEWLINE,
              DEDENT,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          handler = lambda: None  # noop
                                  # (will redefine later)
  
          ################################################################################
  
          # comment 1
          print("I'm done.")
          """,
          [
              NL,
              NAME("handler"),
              OP("="),
              NAME("lambda"),
              OP(":"),
              NAME("None"),
              COMMENT,
              NEWLINE,
              COMMENT,
              NL,
              NL,
              COMMENT,
              NL,
              NL,
              COMMENT,
              NL,
              NAME("print"),
              OP("("),
              STRING,
              OP(")"),
              NEWLINE,
              END,
          ],
      )
      check_code(
          """
          def test3():
              x = 1
          # bad
              print(x)
          """,
          [
              NL,
              NAME("def"),
              NAME("test3"),
              OP("("),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("x"),
              OP("="),
              NUMBER,
              NEWLINE,
              COMMENT,
              NL,
              NAME("print"),
              OP("("),
              NAME("x"),
              OP(")"),
              NEWLINE,
              DEDENT,
              END,
          ],
      )
      check_code(
          """
          class Foo(object):
              #-------------
              def bar(self):
                  if True:
                      pass
  
          # Originally the DEDENTs are all the way down near the decorator. Here we're testing how they'd travel
          # all the way up across multiple comments.
  
          # comment 3
  
          # commmmmmmment 4
          @decorator
          """,
          [
              NL,
              NAME("class"),
              NAME("Foo"),
              OP("("),
              NAME("object"),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              COMMENT,
              NL,
              NAME("def"),
              NAME("bar"),
              OP("("),
              NAME("self"),
              OP(")"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("if"),
              NAME("True"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("pass"),
              NEWLINE,
              DEDENT,
              DEDENT,
              DEDENT,
              NL,
              COMMENT,
              NL,
              COMMENT,
              NL,
              NL,
              COMMENT,
              NL,
              NL,
              COMMENT,
              NL,
              OP("@"),
              NAME("decorator"),
              NEWLINE,
              END,
          ],
      )
  
      # Really, one should avoid code like this.... It won't break the normalizer, but may create problems down
      # the stream.
      check_code(
          """
          if True:
              if False:
                  # INDENT will be inserted before this comment
                  raise
                  # DEDENT will be after this comment
              else:
                  praise()
          """,
          [
              NL,
              NAME("if"),
              NAME("True"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("if"),
              NAME("False"),
              OP(":"),
              NEWLINE,
              INDENT,
              COMMENT,
              NL,
              NAME("raise"),
              NEWLINE,
              COMMENT,
              NL,
              DEDENT,
              NAME("else"),
              OP(":"),
              NEWLINE,
              INDENT,
              NAME("praise"),
              OP("("),
              OP(")"),
              NEWLINE,
              DEDENT,
              DEDENT,
              END,
          ],
      )
  
      for directory in [".", "../../h2o-py/h2o", "../../h2o-py/tests"]:
          absdir = os.path.abspath(directory)
          for dir_name, subdirs, files in os.walk(absdir):
              for f in files:
                  if f.endswith(".py"):
                      filename = os.path.join(dir_name, f)
                      with open(
                          filename, "rt", encoding="utf-8"
                      ) as fff:
                          check_code(fff.read(), filename=filename)
  
file_path: h2o-3/h2o-bindings/bin/pyunit_parser_test.py
imports: import os
  import re
  import textwrap
  import tokenize
  import pyparser
label_desc: Use image node_id 3 for calling a global function with example usage: test_tokenization() and returns: pyparser, tokenize