h2o-bindings/bin/pyparser.py
def _parse1(self):
        """
        First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.
        """
        fragments = []
        tokens = self._tokens

        def advance_after_newline(i0):
            """Return the index of the first token after the end of the current (logical) line."""
            for i in range(i0, len(tokens)):
                if tokens[i].op == NEWLINE:
                    break
            return i + 1

        i = 0
        while i < len(tokens):
            # Assume that we always start at the beginning of a new block
            i0 = i
            tok = tokens[i]
            fragment_type = "???"  # to be determined in the switch clause below

            if tok.op == ENDMARKER:
                fragment_type = "end"
                i += 1
                assert i == len(tokens), "ENDMARKER token encountered before the end of the stream"

            elif tok.op == NL:
                fragment_type = "whitespace"
                # If there are multiple whitespaces, gobble them all
                while tokens[i].op == NL:
                    i += 1

            elif tok.op == COMMENT:
                fragment_type = "comment"
                # Collapse multiple comment lines into a single comment fragment; but only if they are at the same
                # level of indentation.
                is_banner = False
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col == tok.start_col:
                    assert tokens[i + 1].op == NL, "Unexpected token after a comment: %r" % tokens[i + 1]
                    s = tokens[i].str
                    if re.match(r"^#\s?[#*=-]{10,}$", s) or re.match(r"^#\s?[#*=-]{4,}.*?[#*=-]{4,}$", s):
                        is_banner = True
                    i += 2
                if is_banner:
                    fragment_type = "banner-comment"

            elif (tok.op == STRING and tokens[i + 1].op == NEWLINE and
                  all(frag[0] == "whitespace" or frag[0] == "comment" for frag in fragments)):
                i += 2
                fragment_type = "docstring"

            elif tok.op == OP and tok.str == "@" and tokens[i + 1].op == NAME:
                while tokens[i].op == OP and tokens[i].str == "@" and tokens[i + 1].op == NAME:
                    i = advance_after_newline(i)
                fragment_type = "decorator"

            elif tok.op == NAME and tok.str in {"from", "import"}:
                while tokens[i].op == NAME and tokens[i].str in {"from", "import"}:
                    i = advance_after_newline(i)
                fragment_type = "import"

            elif tok.op in {INDENT, DEDENT, NEWLINE}:
                assert False, "Unexpected token %d: %r" % (i, tok)

            else:
                i = advance_after_newline(i)
                if i < len(tokens) and tokens[i].op == INDENT:
                    level = 1
                    while level > 0:
                        i += 1
                        level += tokens[i].indent()
                    assert tokens[i].op == DEDENT
                    i += 1  # consume the last DEDENT
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col > tok.start_col:
                    assert tokens[i + 1].op == NL
                    i += 2
                if tok.op == NAME and tok.str in {"def", "class"}:
                    fragment_type = tok.str
                else:
                    fragment_type = "code"
            assert i > i0, "Stuck at i = %d" % i
            fragments.append((fragment_type, i0, i))
        return fragments
def _parse1(self):
        """
        First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.
        """
        fragments = []
        tokens = self._tokens

        def advance_after_newline(i0):
            """Return the index of the first token after the end of the current (logical) line."""
            for i in range(i0, len(tokens)):
                if tokens[i].op == NEWLINE:
                    break
            return i + 1

        i = 0
        while i < len(tokens):
            # Assume that we always start at the beginning of a new block
            i0 = i
            tok = tokens[i]
            fragment_type = "???"  # to be determined in the switch clause below

            if tok.op == ENDMARKER:
                fragment_type = "end"
                i += 1
                assert i == len(tokens), "ENDMARKER token encountered before the end of the stream"

            elif tok.op == NL:
                fragment_type = "whitespace"
                # If there are multiple whitespaces, gobble them all
                while tokens[i].op == NL:
                    i += 1

            elif tok.op == COMMENT:
                fragment_type = "comment"
                # Collapse multiple comment lines into a single comment fragment; but only if they are at the same
                # level of indentation.
                is_banner = False
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col == tok.start_col:
                    assert tokens[i + 1].op == NL, "Unexpected token after a comment: %r" % tokens[i + 1]
                    s = tokens[i].str
                    if re.match(r"^#\s?[#*=-]{10,}$", s) or re.match(r"^#\s?[#*=-]{4,}.*?[#*=-]{4,}$", s):
                        is_banner = True
                    i += 2
                if is_banner:
                    fragment_type = "banner-comment"

            elif (tok.op == STRING and tokens[i + 1].op == NEWLINE and
                  all(frag[0] == "whitespace" or frag[0] == "comment" for frag in fragments)):
                i += 2
                fragment_type = "docstring"

            elif tok.op == OP and tok.str == "@" and tokens[i + 1].op == NAME:
                while tokens[i].op == OP and tokens[i].str == "@" and tokens[i + 1].op == NAME:
                    i = advance_after_newline(i)
                fragment_type = "decorator"

            elif tok.op == NAME and tok.str in {"from", "import"}:
                while tokens[i].op == NAME and tokens[i].str in {"from", "import"}:
                    i = advance_after_newline(i)
                fragment_type = "import"

            elif tok.op in {INDENT, DEDENT, NEWLINE}:
                assert False, "Unexpected token %d: %r" % (i, tok)

            else:
                i = advance_after_newline(i)
                if i < len(tokens) and tokens[i].op == INDENT:
                    level = 1
                    while level > 0:
                        i += 1
                        level += tokens[i].indent()
                    assert tokens[i].op == DEDENT
                    i += 1  # consume the last DEDENT
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col > tok.start_col:
                    assert tokens[i + 1].op == NL
                    i += 2
                if tok.op == NAME and tok.str in {"def", "class"}:
                    fragment_type = tok.str
                else:
                    fragment_type = "code"
            assert i > i0, "Stuck at i = %d" % i
            fragments.append((fragment_type, i0, i))
        return fragments
def _parse1 ( self ) : fragments = [ ] tokens = self . _tokens def advance_after_newline ( i0 ) : """Return the index of the first token after the end of the current (logical) line.""" for i in range ( i0 , len ( tokens ) ) : if tokens [ i ] . op == NEWLINE : break return i + 1 i = 0 while i < len ( tokens ) : # Assume that we always start at the beginning of a new block i0 = i tok = tokens [ i ] fragment_type = "???" # to be determined in the switch clause below if tok . op == ENDMARKER : fragment_type = "end" i += 1 assert i == len ( tokens ) , "ENDMARKER token encountered before the end of the stream" elif tok . op == NL : fragment_type = "whitespace" # If there are multiple whitespaces, gobble them all while tokens [ i ] . op == NL : i += 1 elif tok . op == COMMENT : fragment_type = "comment" # Collapse multiple comment lines into a single comment fragment; but only if they are at the same # level of indentation. is_banner = False while i < len ( tokens ) and tokens [ i ] . op == COMMENT and tokens [ i ] . start_col == tok . start_col : assert tokens [ i + 1 ] . op == NL , "Unexpected token after a comment: %r" % tokens [ i + 1 ] s = tokens [ i ] . str if re . match ( r"^#\s?[#*=-]{10,}$" , s ) or re . match ( r"^#\s?[#*=-]{4,}.*?[#*=-]{4,}$" , s ) : is_banner = True i += 2 if is_banner : fragment_type = "banner-comment" elif ( tok . op == STRING and tokens [ i + 1 ] . op == NEWLINE and all ( frag [ 0 ] == "whitespace" or frag [ 0 ] == "comment" for frag in fragments ) ) : i += 2 fragment_type = "docstring" elif tok . op == OP and tok . str == "@" and tokens [ i + 1 ] . op == NAME : while tokens [ i ] . op == OP and tokens [ i ] . str == "@" and tokens [ i + 1 ] . op == NAME : i = advance_after_newline ( i ) fragment_type = "decorator" elif tok . op == NAME and tok . str in { "from" , "import" } : while tokens [ i ] . op == NAME and tokens [ i ] . str in { "from" , "import" } : i = advance_after_newline ( i ) fragment_type = "import" elif tok . op in { INDENT , DEDENT , NEWLINE } : assert False , "Unexpected token %d: %r" % ( i , tok ) else : i = advance_after_newline ( i ) if i < len ( tokens ) and tokens [ i ] . op == INDENT : level = 1 while level > 0 : i += 1 level += tokens [ i ] . indent ( ) assert tokens [ i ] . op == DEDENT i += 1 # consume the last DEDENT while i < len ( tokens ) and tokens [ i ] . op == COMMENT and tokens [ i ] . start_col > tok . start_col : assert tokens [ i + 1 ] . op == NL i += 2 if tok . op == NAME and tok . str in { "def" , "class" } : fragment_type = tok . str else : fragment_type = "code" assert i > i0 , "Stuck at i = %d" % i fragments . append ( ( fragment_type , i0 , i ) ) return fragments
First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.
First stage of parsing the code ( stored as a raw stream of tokens ) .