Update links to discontinued W3C document with a spec

http://www.w3.org/TR/html-markup is marked as discontinued

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\r\n' | '\n' | '\r';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    whitespace_or_newline = whitespace | newline;

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    action advance_newline {
        advance_line(1)
    }

    action hold_and_return {
        fhold;
        fret;
    }

    # Comments
    #
    # http://www.w3.org/TR/html/syntax.html#comments
    #
    # Unlike the W3C specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html/syntax.html#the-doctype
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i (whitespace_or_newline+ $count_newlines);

    action start_doctype {
        callback_simple(id_on_doctype_start);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }

        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };

        newline => advance_newline;

        whitespace;
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Characters that can be used for unquoted HTML attribute values.
    # See https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example
    # for more info.
    html_unquoted_value = ^(
        squote | dquote | '`' | '=' | '<' | '>' | whitespace_or_newline
    )+;

    # Machine used for processing HTML attribute values.
    html_attribute_value := |*
        squote | dquote => {
            fhold;
            fnext xml_attribute_value;
        };

        # Unquoted attribute values are lexed as if they were single quoted
        # strings.
        html_unquoted_value => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_and_return;
    *|;

    # Machine used for processing XML attribute values.
    xml_attribute_value := |*
        # The following two actions use "fnext" instead of "fcall". Combined
        # with "element_head" using "fcall" to jump to this machine this means
        # we can return back to "element_head" after processing a single string.
        squote => {
            callback_simple(id_on_string_squote);

            fnext string_squote;
        };

        dquote => {
            callback_simple(id_on_string_dquote);

            fnext string_dquote;
        };

        any => hold_and_return;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        newline => advance_newline;

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            if ( html_p )
            {
                fcall html_attribute_value;
            }
            else
            {
                fcall xml_attribute_value;
            }
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( literal_html_element_p() )
            {
                fnext literal_html_element;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };

        any;
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html/syntax.html#text
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # Certain tags in HTML can contain basically anything except for the literal
    # closing tag. Two examples are script and style tags.  As a result of this
    # we can't use the regular text machine.
    literal_html_closing_tags = '</script>' | '</style>';
    literal_html_allowed = (any* -- literal_html_closing_tags) $count_newlines;

    literal_html_element := |*
        literal_html_allowed => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        literal_html_allowed %{ mark = p; } literal_html_closing_tags => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment_start  => start_comment;
        cdata_start    => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\r\n' | '\n' | '\r';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    whitespace_or_newline = whitespace | newline;

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    action advance_newline {
        advance_line(1)
    }

    action hold_and_return {
        fhold;
        fret;
    }

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i (whitespace_or_newline+ $count_newlines);

    action start_doctype {
        callback_simple(id_on_doctype_start);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }

        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };

        newline => advance_newline;

        whitespace;
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Characters that can be used for unquoted HTML attribute values.
    # See https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example
    # for more info.
    html_unquoted_value = ^(
        squote | dquote | '`' | '=' | '<' | '>' | whitespace_or_newline
    )+;

    # Machine used for processing HTML attribute values.
    html_attribute_value := |*
        squote | dquote => {
            fhold;
            fnext xml_attribute_value;
        };

        # Unquoted attribute values are lexed as if they were single quoted
        # strings.
        html_unquoted_value => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_and_return;
    *|;

    # Machine used for processing XML attribute values.
    xml_attribute_value := |*
        # The following two actions use "fnext" instead of "fcall". Combined
        # with "element_head" using "fcall" to jump to this machine this means
        # we can return back to "element_head" after processing a single string.
        squote => {
            callback_simple(id_on_string_squote);

            fnext string_squote;
        };

        dquote => {
            callback_simple(id_on_string_dquote);

            fnext string_dquote;
        };

        any => hold_and_return;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        newline => advance_newline;

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            if ( html_p )
            {
                fcall html_attribute_value;
            }
            else
            {
                fcall xml_attribute_value;
            }
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( literal_html_element_p() )
            {
                fnext literal_html_element;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };

        any;
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # Certain tags in HTML can contain basically anything except for the literal
    # closing tag. Two examples are script and style tags.  As a result of this
    # we can't use the regular text machine.
    literal_html_closing_tags = '</script>' | '</style>';
    literal_html_allowed = (any* -- literal_html_closing_tags) $count_newlines;

    literal_html_element := |*
        literal_html_allowed => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        literal_html_allowed %{ mark = p; } literal_html_closing_tags => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment_start  => start_comment;
        cdata_start    => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%


Use the variable feature for pullscan.

Use the variable feature for pullscan.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFSIZE 4096

typedef struct _Scanner {
	/* Scanner state. */
    int cs;
    int act;
    int have;
    int curline;
    char *tokstart;
    char *tokend;
    char *p;
    char *pe;
	FILE *file;
	int done;

	/* Token data */
	char *data;
	int len;
    int value;

	char buf[BUFSIZE];
} Scanner;

%%{
	machine Scanner;
	write data;
}%%

void scan_init( Scanner *s, FILE *file )
{
	memset (s, '\0', sizeof(Scanner));
	s->curline = 1;
	s->file = file;
	%% write init;
}

#define TK_NO_TOKEN (-1)
#define TK_ERR 128
#define TK_EOF 129
#define TK_Identifier 130
#define TK_Number 131
#define TK_String 132

#define ret_tok( _tok ) token = _tok; s->data = s->tokstart

int scan( Scanner *s )
{
	int token = TK_NO_TOKEN;
	int space, readlen;

	while ( 1 ) {
		if ( s->p == s->pe ) {
			printf("scanner: need more data\n");

			if ( s->tokstart == 0 )
				s->have = 0;
			else {
				/* There is data that needs to be shifted over. */
				printf("scanner: buffer broken mid token\n");
				s->have = s->pe - s->tokstart;
				memmove( s->buf, s->tokstart, s->have );
				s->tokend -= (s->tokstart-s->buf);
				s->tokstart = s->buf;
			}

			s->p = s->buf + s->have;
			space = BUFSIZE - s->have;

			if ( space == 0 ) {
				/* We filled up the buffer trying to scan a token. */
				printf("scanner: out of buffer space\n");
				return TK_ERR;
			}

			if ( s->done ) {
				printf("scanner: end of file\n");
				s->p[0] = 0;
				readlen = 1;
			}
			else {
				readlen = fread( s->p, 1, space, s->file );
				if ( readlen < space )
					s->done = 1;
			}

			s->pe = s->p + readlen;
		}

		%%{
			machine Scanner;
			access s->;
			variable p s->p;
			variable pe s->pe;

			main := |*

			# Identifiers
			( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
				{ ret_tok( TK_Identifier ); fbreak; };

			# Whitespace
			[ \t\n];

			'"' ( [^\\"] | '\\' any ) * '"' =>
				{ ret_tok( TK_String ); fbreak; };

			# Number
			digit+ =>
				{ ret_tok( TK_Number ); fbreak; };

			# EOF
			0 =>
				{ ret_tok( TK_EOF ); fbreak; };

			# Anything else
			any =>
				{ ret_tok( *s->p ); fbreak; };

			*|;

			write exec;
		}%%

		if ( s->cs == Scanner_error )
			return TK_ERR;

		if ( token != TK_NO_TOKEN ) {
			/* Save p and pe. fbreak does not advance p. */
			s->p += 1;
			s->len = s->p - s->data;
			return token;
		}
	}
}

int main (int argc, char** argv)
{
	Scanner ss;
	int tok;

	scan_init(&ss, stdin);

	while ( 1 ) {
		tok = scan (&ss);
		if ( tok == TK_EOF ) {
			printf ("parser: EOF\n");
			break;
		}
		else if ( tok == TK_ERR ) {
			printf ("parser: ERR\n");
			break;
		}
		else {
			printf ("parser: %d \"", tok);
			fwrite ( ss.data, 1, ss.len, stdout );
			printf ("\"\n" );
		}
	}

	return 0;
}

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFSIZE 4096

typedef struct _Scanner {
	/* Scanner state. */
    int cs;
    int act;
    int have;
    int curline;
    char *tokstart;
    char *tokend;
    char *p;
    char *pe;
	FILE *file;
	int done;

	/* Token data */
	char *data;
	int len;
    int value;

	char buf[BUFSIZE];
} Scanner;

%%{
	machine Scanner;
	write data;
}%%

void scan_init( Scanner *s, FILE *file )
{
	memset (s, '\0', sizeof(Scanner));
	s->curline = 1;
	s->file = file;
	%% write init;
}

#define TK_NO_TOKEN (-1)
#define TK_ERR 128
#define TK_EOF 129
#define TK_Identifier 130
#define TK_Number 131

#define ret_tok( _tok ) token = _tok; s->data = s->tokstart

int scan( Scanner *s )
{
	char *p = s->p;
	char *pe = s->pe;
	int token = TK_NO_TOKEN;
	int space, readlen;

	while ( 1 ) {
		if ( p == pe ) {
			printf("scanner: need more data\n");

			if ( s->tokstart == 0 )
				s->have = 0;
			else {
				/* There is data that needs to be shifted over. */
				printf("scanner: buffer broken mid token\n");
				s->have = pe - s->tokstart;
				memmove( s->buf, s->tokstart, s->have );
				s->tokend -= (s->tokstart-s->buf);
				s->tokstart = s->buf;
			}

			p = s->buf + s->have;
			space = BUFSIZE - s->have;

			if ( space == 0 ) {
				/* We filled up the buffer trying to scan a token. */
				printf("scanner: out of buffer space\n");
				return TK_ERR;
			}

			if ( s->done ) {
				printf("scanner: end of file\n");
				p[0] = 0;
				readlen = 1;
			}
			else {
				readlen = fread( p, 1, space, s->file );
				if ( readlen < space )
					s->done = 1;
			}

			pe = p + readlen;
		}

		%%{
			machine Scanner;
			access s->;

			main := |*

			# Identifiers
			( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
				{ ret_tok( TK_Identifier ); fbreak; };

			# Whitespace
			[ \t\n];

			'"' ( [^\\"] | '\\' any ) * '"';

			# Number
			digit+ =>
				{ ret_tok( TK_Number ); fbreak; };

			# EOF
			0 =>
				{ ret_tok( TK_EOF ); fbreak; };

			# Anything else
			any =>
				{ ret_tok( *p ); fbreak; };

			*|;

			write exec;
		}%%

		if ( s->cs == Scanner_error )
			return TK_ERR;

		if ( token != TK_NO_TOKEN ) {
			/* Save p and pe. fbreak does not advance p. */
			s->p = p + 1;
			s->pe = pe;
			s->len = s->p - s->data;
			return token;
		}
	}
}

int main (int argc, char** argv)
{
	Scanner ss;
	int tok;

	scan_init(&ss, stdin);

	while ( 1 ) {
		tok = scan (&ss);
		if ( tok == TK_EOF ) {
			printf ("parser: EOF\n");
			break;
		}
		else if ( tok == TK_ERR ) {
			printf ("parser: ERR\n");
			break;
		}
		else {
			printf ("parser: %d \"", tok);
			fwrite ( ss.data, 1, ss.len, stdout );
			printf ("\"\n" );
		}
	}

	return 0;
}


http: Sync response parser with request parser

This fix mostly contains the changes from respective fix to request
parser int efd61d38 (http: fix the request parser). As heading part of
requests and responses coinside, the changes make sense.

Also it excludes the heading space from 'header: value' string which
is for some reason was present in the request parser from the very
beginning.

Signed-off-by: Pavel Emelyanov <c9a32589e048e044184536f7ac71ef92fe82df3e@scylladb.com>

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#include <seastar/core/ragel.hh>
#include <memory>
#include <unordered_map>

namespace seastar {

struct http_response {
    sstring _version;
    std::unordered_map<sstring, sstring> _headers;
    int _status_code;
};

%% machine http_response;

%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action store_version {
    _rsp->_version = str();
}

action store_field_name {
    _field_name = str();
}

action store_value {
    _value = str();
}

action no_mark_store_value {
    _value = get_str();
    g.mark_start(nullptr);
}

action checkpoint {
    // Needs a start to be marked beforehand.
    // Used to mark the candidate end of value string. Can be moved furhter by repetitive use
    // of this action.
    // To store the string that ends on the last checkpoint (instead of the last processed character)
    // use %no_mark_store_value instead of %store_value
    g.mark_end(p);
    g.mark_start(p);
}

action assign_field {
    if (_rsp->_headers.count(_field_name)) {
        // RFC 7230, section 3.2.2.  Field Parsing:
        // A recipient MAY combine multiple header fields with the same field name into one
        // "field-name: field-value" pair, without changing the semantics of the message,
        // by appending each subsequent field value to the combined field value in order, separated by a comma.
        _rsp->_headers[_field_name] += sstring(",") + std::move(_value);
    } else {
        _rsp->_headers[_field_name] = std::move(_value);
    }
}

action extend_field  {
    // RFC 7230, section 3.2.4.  Field Order:
    // A server that receives an obs-fold in a request message that is not
    // within a message/http container MUST either reject the message [...]
    // or replace each received obs-fold with one or more SP octets [...]
    _rsp->_headers[_field_name] += sstring(" ") + std::move(_value);
}

action store_status {
    _rsp->_status_code = std::atoi(str().c_str());
}

action done {
    done = true;
    fbreak;
}

cr = '\r';
lf = '\n';
crlf = '\r\n';
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';

sp = ' ';
ht = '\t';

sp_ht = sp | ht;

http_version = 'HTTP/' (digit '.' digit) >mark %store_version;

obs_text = 0x80..0xFF; # defined in RFC 7230, Section 3.2.6.
field_vchars = (graph | obs_text)+ %checkpoint;
field_content = (field_vchars sp_ht*)*;

field = tchar+ >mark %store_field_name;
value = field_content >mark %no_mark_store_value;
status_code = (digit digit digit) >mark %store_status;
start_line = http_version space status_code space (any - cr - lf)* crlf;
header_1st = (field ':' sp_ht* value crlf) %assign_field;
header_cont = (sp_ht+ value crlf) %extend_field;
header = header_1st header_cont*;
main := start_line header* (crlf @done);

}%%

class http_response_parser : public ragel_parser_base<http_response_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unique_ptr<http_response> _rsp;
    sstring _field_name;
    sstring _value;
    state _state;
public:
    void init() {
        init_base();
        _rsp.reset(new http_response());
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            if (p == eof) {
                _state = state::eof;
            } else if (p != pe) {
                _state = state::error;
            } else {
                p = nullptr;
            }
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_response() {
        return std::move(_rsp);
    }
    bool eof() const {
        return _state == state::eof;
    }
    bool failed() const {
        return _state == state::error;
    }
};

}

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#include <seastar/core/ragel.hh>
#include <memory>
#include <unordered_map>

namespace seastar {

struct http_response {
    sstring _version;
    std::unordered_map<sstring, sstring> _headers;
    int _status_code;
};

%% machine http_response;

%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action store_version {
    _rsp->_version = str();
}

action store_field_name {
    _field_name = str();
}

action store_value {
    _value = str();
}

action assign_field {
    _rsp->_headers[_field_name] = std::move(_value);
}

action extend_field  {
    _rsp->_headers[_field_name] += sstring(" ") + std::move(_value);
}

action store_status {
    _rsp->_status_code = std::atoi(str().c_str());
}

action done {
    done = true;
    fbreak;
}

cr = '\r';
lf = '\n';
crlf = '\r\n';
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';

sp = ' ';
ht = '\t';

sp_ht = sp | ht;

http_version = 'HTTP/' (digit '.' digit) >mark %store_version;

field = tchar+ >mark %store_field_name;
value = any* >mark %store_value;
status_code = (digit digit digit) >mark %store_status;
start_line = http_version space status_code space (any - cr - lf)* crlf;
header_1st = (field sp_ht* ':' value :> crlf) %assign_field;
header_cont = (sp_ht+ value sp_ht* crlf) %extend_field;
header = header_1st header_cont*;
main := start_line header* :> (crlf @done);

}%%

class http_response_parser : public ragel_parser_base<http_response_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unique_ptr<http_response> _rsp;
    sstring _field_name;
    sstring _value;
    state _state;
public:
    void init() {
        init_base();
        _rsp.reset(new http_response());
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            p = nullptr;
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_response() {
        return std::move(_rsp);
    }
    bool eof() const {
        return _state == state::eof;
    }
};

}