Patch to Oga Upstream to fix "Oga Parser doesn't handle dots in XML Tags". Fixes rubyjedi/soap4r#7

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\r\n' | '\n' | '\r';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_\.];
    identifier = ident_char+;

    whitespace_or_newline = whitespace | newline;

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    action advance_newline {
        advance_line(1);
    }

    action hold_and_return {
        fhold;
        fret;
    }

    # Comments
    #
    # http://www.w3.org/TR/html/syntax.html#comments
    #
    # Unlike the W3C specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html/syntax.html#the-doctype
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i (whitespace_or_newline+ $count_newlines);

    action start_doctype {
        callback_simple(id_on_doctype_start);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }

        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };

        newline => advance_newline;

        whitespace;
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    action start_element {
        fhold;
        fnext element_name;
    }

    action start_close_element {
        fnext element_close;
    }

    action close_element {
        callback(id_on_element_end, data, encoding, ts, te);
    }

    action close_element_fnext_main {
        callback_simple(id_on_element_end);

        fnext main;
    }

    element_start = '<' ident_char;
    element_end   = '</';

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for lexing the closing tag of an element
    element_close := |*
        # namespace prefixes, currently not used but allows the rule below it
        # to be used for the actual element name.
        identifier ':';

        identifier => close_element;

        '>' => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        any $count_newlines;
    *|;

    # Machine used after matching the "=" of an attribute and just before moving
    # into the actual attribute value.
    attribute_pre := |*
        whitespace_or_newline $count_newlines;

        squote | dquote => {
            fhold;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext quoted_attribute_value;
        };

        any => {
            fhold;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            if ( html_p )
            {
                fnext unquoted_attribute_value;
            }
            /* XML doesn't support unquoted attribute values */
            else
            {
                fret;
            }
        };
    *|;

    # Machine for processing unquoted HTML attribute values.
    #
    # The HTML specification describes a set of characters that can be allowed
    # in an unquoted value at https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example.
    #
    # As is always the case with HTML everybody completely ignores this
    # specification and thus every library and browser out these is expected to
    # support input such as `.
    #
    # Oga too has to support this, thus the only characters it disallows in
    # unquoted attribute values are:
    #
    # * > (used for terminating open tags)
    # * whitespace
    #
    unquoted_attribute_value := |*
        ^('>' | whitespace_or_newline)+ => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_and_return;
    *|;

    # Machine used for processing quoted XML/HTML attribute values.
    quoted_attribute_value := |*
        # The following two actions use "fnext" instead of "fcall". Combined
        # with "element_head" using "fcall" to jump to this machine this means
        # we can return back to "element_head" after processing a single string.
        squote => {
            callback_simple(id_on_string_squote);

            fnext string_squote;
        };

        dquote => {
            callback_simple(id_on_string_dquote);

            fnext string_dquote;
        };

        any => hold_and_return;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        newline => advance_newline;

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            fcall attribute_pre;
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( html_script_p() )
            {
                fnext html_script;
            }
            else if ( html_style_p() )
            {
                fnext html_style;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };

        any;
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html/syntax.html#text
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\r\n' | '\n' | '\r';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    whitespace_or_newline = whitespace | newline;

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    action advance_newline {
        advance_line(1);
    }

    action hold_and_return {
        fhold;
        fret;
    }

    # Comments
    #
    # http://www.w3.org/TR/html/syntax.html#comments
    #
    # Unlike the W3C specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html/syntax.html#the-doctype
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i (whitespace_or_newline+ $count_newlines);

    action start_doctype {
        callback_simple(id_on_doctype_start);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }

        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };

        newline => advance_newline;

        whitespace;
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    action start_element {
        fhold;
        fnext element_name;
    }

    action start_close_element {
        fnext element_close;
    }

    action close_element {
        callback(id_on_element_end, data, encoding, ts, te);
    }

    action close_element_fnext_main {
        callback_simple(id_on_element_end);

        fnext main;
    }

    element_start = '<' ident_char;
    element_end   = '</';

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for lexing the closing tag of an element
    element_close := |*
        # namespace prefixes, currently not used but allows the rule below it
        # to be used for the actual element name.
        identifier ':';

        identifier => close_element;

        '>' => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        any $count_newlines;
    *|;

    # Machine used after matching the "=" of an attribute and just before moving
    # into the actual attribute value.
    attribute_pre := |*
        whitespace_or_newline $count_newlines;

        squote | dquote => {
            fhold;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext quoted_attribute_value;
        };

        any => {
            fhold;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            if ( html_p )
            {
                fnext unquoted_attribute_value;
            }
            /* XML doesn't support unquoted attribute values */
            else
            {
                fret;
            }
        };
    *|;

    # Machine for processing unquoted HTML attribute values.
    #
    # The HTML specification describes a set of characters that can be allowed
    # in an unquoted value at https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example.
    #
    # As is always the case with HTML everybody completely ignores this
    # specification and thus every library and browser out these is expected to
    # support input such as `.
    #
    # Oga too has to support this, thus the only characters it disallows in
    # unquoted attribute values are:
    #
    # * > (used for terminating open tags)
    # * whitespace
    #
    unquoted_attribute_value := |*
        ^('>' | whitespace_or_newline)+ => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_and_return;
    *|;

    # Machine used for processing quoted XML/HTML attribute values.
    quoted_attribute_value := |*
        # The following two actions use "fnext" instead of "fcall". Combined
        # with "element_head" using "fcall" to jump to this machine this means
        # we can return back to "element_head" after processing a single string.
        squote => {
            callback_simple(id_on_string_squote);

            fnext string_squote;
        };

        dquote => {
            callback_simple(id_on_string_dquote);

            fnext string_dquote;
        };

        any => hold_and_return;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        newline => advance_newline;

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            fcall attribute_pre;
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( html_script_p() )
            {
                fnext html_script;
            }
            else if ( html_style_p() )
            {
                fnext html_style;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };

        any;
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html/syntax.html#text
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "


Bizarre UTF-8 bug

Bizarre UTF-8 bug

module Gherkin
  module Parser
    class Table
      %%{
        machine table;

        action initialize {
          current_row = []
          @begin_row = p
        }

        action begin_content {
          @content_start = p
        }

        action store_row {
          @rows << current_row
        }

        action store_cell_content {
          con = data[@content_start...p].pack("U*").strip
          current_row << (con.empty? ? nil : con)
        }

        action no_content {
          current_row << nil
        }

        action store_table {
          if @rows.size != 0
            @listener.table(@rows, @line)
          end
        }

        action bad_table_row {
          con = data[@begin_row...p].pack("U*").strip
          # @listener.table_error("Unclosed table row", con, @bad_row_line)
          raise ParsingError.new(@bad_row_line)
        }

        action inc_line_number {
          @cur_line += 1
        }

        action set_bad_table_row_line {
          @bad_row_line = @cur_line
        }

        include table_common "table_common.rl";
      }%%

      def initialize(listener,line)
        @line = line
        @cur_line = line
        @listener = listener
        %% write data;
      end

      def scan(data)
        @rows = []
        data = data.unpack("U*") if data.is_a?(String)
        eof = data.size

        %% write init;
        %% write exec;
      end
    end
  end
end

module Gherkin
  module Parser
    class Table
      %%{
        machine table;

        action initialize {
          current_row = []
          @begin_row = p
        }

        action begin_content {
          @content_start = p
        }

        action store_row {
          @rows << current_row
        }

        action store_cell_content {
          con = data[@content_start...p].pack("U*").strip
          current_row << (con.empty? ? nil : con)
        }

        action no_content {
          current_row << nil
        }

        action store_table {
          if @rows.size != 0
            @listener.table(@rows, @line)
          end
        }

        action bad_table_row {
          con = data[@begin_row...p].pack("U*").strip
          @listener.table_error("Unclosed table row", con, @bad_row_line)
        }

        action inc_line_number {
          @cur_line += 1
        }

        action set_bad_table_row_line {
          @bad_row_line = @cur_line
        }

        include table_common "table_common.rl";
      }%%

      def initialize(listener,line)
        @line = line
        @cur_line = line
        @listener = listener
        %% write data;
      end

      def scan(data)
        @rows = []
        data = data.unpack("U*") if data.is_a?(String)
        eof = data.size

        %% write init;
        %% write exec;
      end
    end
  end
end


accept case-insensitive transfer encoding header

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#include <limits.h>
#include <assert.h>

/* parser->flags */
#define EATING      0x01
#define ERROR       0x02
#define CHUNKED     0x04
#define EAT_FOREVER 0x10

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };

#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))

#undef NULL
#define NULL ((void*)(0))

#define MAX_FIELD_SIZE (80*1024)

#define REMAINING (unsigned long)(pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->flags |= ERROR;                                        \
      return;                                                        \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
    if (callback_return_value != 0) {                                \
      parser->flags |= ERROR;                                        \
      return;                                                        \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
  parser->chunk_size = 0;                                            \
  parser->flags = 0;                                                 \
  parser->header_field_mark = NULL;                                  \
  parser->header_value_mark = NULL;                                  \
  parser->query_string_mark = NULL;                                  \
  parser->path_mark = NULL;                                          \
  parser->uri_mark = NULL;                                           \
  parser->fragment_mark = NULL;                                      \
  parser->status_code = 0;                                           \
  parser->method = 0;                                                \
  parser->version = HTTP_VERSION_OTHER;                              \
  parser->keep_alive = -1;                                           \
  parser->content_length = -1;                                       \
  parser->body_read = 0

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->flags &= ~EATING;                                      \
      if (!(parser->flags & CHUNKED)) {                              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->flags |= EATING;                                       \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }
    }
  }

  action content_length {
    if (parser->content_length == -1) parser->content_length = 0;
    if (parser->content_length > INT_MAX) {
      parser->flags |= ERROR;
      return;
    }
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_chunked_encoding { parser->flags |= CHUNKED;  }

  action set_keep_alive { parser->keep_alive = 1; }
  action set_not_keep_alive { parser->keep_alive = 0; }

  action version_11 { parser->version = HTTP_VERSION_11; }
  action version_10 { parser->version = HTTP_VERSION_10; }
  action version_09 { parser->version = HTTP_VERSION_09; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return;
    }

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->flags & CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with
       * skip_chunk_data */
      if (parser->content_length < 0) {
        /* If we didn't get a content length; if not keep-alive
         * just read body until EOF */
        if (!http_parser_should_keep_alive(parser)) {
          parser->flags |= EAT_FOREVER;
          parser->chunk_size = REMAINING;
        } else {
          /* Otherwise, if keep-alive, then assume the message
           * has no body. */
          parser->chunk_size = parser->content_length = 0;
        }
      } else {
        parser->chunk_size = parser->content_length;
      }
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->chunk_size));

      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape | "\"");
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" ( "1.1" %version_11
                         | "1.0" %version_10
                         | "0.9" %version_09
                         | (digit "." digit)
                         );

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding"i hsep "chunked"i %use_chunked_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii* -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode (" " ReasonPhrase)? CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ ( chunk_extension | " "+) CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit* ) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size ( chunk_extension | " "+ ) CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_fragment = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
void
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe, *eof;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;
  eof = len ? NULL : pe;

  if (parser->flags & EAT_FOREVER) {
    if (len == 0) {
      if (parser->on_message_complete) {
        callback_return_value = parser->on_message_complete(parser);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    } else {
      if (parser->on_body) {
        callback_return_value = parser->on_body(parser, p, len);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    }
    return;
  }

  if (0 < parser->chunk_size && (parser->flags & EATING)) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return;
    }
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

  assert(p <= pe && "buffer overflow after parsing execute");
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->flags & ERROR) return 1;
  return parser->cs == http_parser_error;
}

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#include <limits.h>
#include <assert.h>

/* parser->flags */
#define EATING      0x01
#define ERROR       0x02
#define CHUNKED     0x04
#define EAT_FOREVER 0x10

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };

#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))

#undef NULL
#define NULL ((void*)(0))

#define MAX_FIELD_SIZE (80*1024)

#define REMAINING (unsigned long)(pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->flags |= ERROR;                                        \
      return;                                                        \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
    if (callback_return_value != 0) {                                \
      parser->flags |= ERROR;                                        \
      return;                                                        \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
  parser->chunk_size = 0;                                            \
  parser->flags = 0;                                                 \
  parser->header_field_mark = NULL;                                  \
  parser->header_value_mark = NULL;                                  \
  parser->query_string_mark = NULL;                                  \
  parser->path_mark = NULL;                                          \
  parser->uri_mark = NULL;                                           \
  parser->fragment_mark = NULL;                                      \
  parser->status_code = 0;                                           \
  parser->method = 0;                                                \
  parser->version = HTTP_VERSION_OTHER;                              \
  parser->keep_alive = -1;                                           \
  parser->content_length = -1;                                       \
  parser->body_read = 0

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->flags &= ~EATING;                                      \
      if (!(parser->flags & CHUNKED)) {                              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->flags |= EATING;                                       \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }
    }
  }

  action content_length {
    if (parser->content_length == -1) parser->content_length = 0;
    if (parser->content_length > INT_MAX) {
      parser->flags |= ERROR;
      return;
    }
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_chunked_encoding { parser->flags |= CHUNKED;  }

  action set_keep_alive { parser->keep_alive = 1; }
  action set_not_keep_alive { parser->keep_alive = 0; }

  action version_11 { parser->version = HTTP_VERSION_11; }
  action version_10 { parser->version = HTTP_VERSION_10; }
  action version_09 { parser->version = HTTP_VERSION_09; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return;
    }

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->flags & CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with
       * skip_chunk_data */
      if (parser->content_length < 0) {
        /* If we didn't get a content length; if not keep-alive
         * just read body until EOF */
        if (!http_parser_should_keep_alive(parser)) {
          parser->flags |= EAT_FOREVER;
          parser->chunk_size = REMAINING;
        } else {
          /* Otherwise, if keep-alive, then assume the message
           * has no body. */
          parser->chunk_size = parser->content_length = 0;
        }
      } else {
        parser->chunk_size = parser->content_length;
      }
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->chunk_size));

      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return;
      }

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape | "\"");
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" ( "1.1" %version_11
                         | "1.0" %version_10
                         | "0.9" %version_09
                         | (digit "." digit)
                         );

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding" hsep "chunked"i %use_chunked_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii* -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode (" " ReasonPhrase)? CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ ( chunk_extension | " "+) CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit* ) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size ( chunk_extension | " "+ ) CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_fragment = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
void
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe, *eof;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;
  eof = len ? NULL : pe;

  if (parser->flags & EAT_FOREVER) {
    if (len == 0) {
      if (parser->on_message_complete) {
        callback_return_value = parser->on_message_complete(parser);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    } else {
      if (parser->on_body) {
        callback_return_value = parser->on_body(parser, p, len);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    }
    return;
  }

  if (0 < parser->chunk_size && (parser->flags & EATING)) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return;
    }
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

  assert(p <= pe && "buffer overflow after parsing execute");
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->flags & ERROR) return 1;
  return parser->cs == http_parser_error;
}


improve type parsing error msg

improve type parsing error msg

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

#define PARSE_ERROR(...) do { \
  snprintf(err_buf, err_buf_size, __VA_ARGS__); \
  err = 1; \
  goto bail; \
} while(0);

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays (line %d)", curr_line);

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld (line %d)", curr_item_index, curr_line);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld (line %d)", i, curr_line);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing (line %d)", curr_line);

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type (line %d)", curr_line); }
           array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error (line %d)", curr_line);

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

#define PARSE_ERROR(...) do { \
  snprintf(err_buf, err_buf_size, __VA_ARGS__); \
  err = 1; \
  goto bail; \
} while(0);

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays (line %d)", curr_line);

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld (line %d)", curr_item_index, curr_line);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld (line %d)", i, curr_line);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing (line %d)", curr_line);

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error (line %d)", curr_line);

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}


Fix closing </pre> being chopped off

Fix closing </pre> being chopped off

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#define superredcloth_scan_c

#include <ruby.h>
#include "superredcloth.h"

VALUE super_ParseError, super_RedCloth, super_HTML;

%%{

  machine superredcloth_scan;
  include superredcloth_common "ext/superredcloth_scan/superredcloth_common.rl";

  action notextile { rb_str_append(html, rb_funcall(rb_formatter, rb_intern("ignore"), 1, regs)); }
  action extend { plain_block = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }
  action no_extend { plain_block = rb_str_new2("p"); }

  # blocks
  notextile_start = "<notextile>" ;
  notextile_end = "</notextile>" ;
  pre_start = "<pre" [^>]* ">" ;
  pre_end = "</pre>" ;
  bc_start = ( "bc" A C :> "." ( "." %extend | "" %no_extend ) " "+ ) ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "bq" | "bc" | "pre" | "notextile" | "div" ) >A %{ STORE(type) } ;
  block_start = ( btype A C :> "." ( "." %extend | "" %no_extend ) " "+ ) ;
  block_end = ( CRLF{2} | EOF );
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  footnote_start = ( ftype A C :> dotspace ) ;
  ul = "*" %{nest++; list_type = "ul";};
  ol = "#" %{nest++; list_type = "ol";};
  list_start  = ( ( ul | ol )+ N A C :> " "+ ) >{nest = 0;} ;

  # tables
  para = ( default+ ) -- CRLF ;
  btext = para ( CRLF{2} )? ;
  tddef = ( S A C D? :> dotspace ) ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, text, td);} ) >X ;
  trdef = ( A C :> dotspace ) ;
  tr = ( trdef? "|" %{INLINE(table, tr_open);} td+ ) >X %{INLINE(table, tr_close);} ;
  trows = ( tr (CRLF >X tr)* ) ;
  tdef = ( "table" >X A C :> dotspace CRLF ) ;
  table = ( tdef? trows >{INLINE(table, table_open);} ) >{ reg = NULL; } ;

  bc := |*
    block_end       { ADD_BLOCKCODE(); fgoto main; };
    default => cat;
  *|;

  pre := |*
    pre_end         { CAT(block); DONE(block); fgoto main; };
    default => cat;
  *|;

  notextile := |*
    notextile_end   { DONE(block); fgoto main; };
    default => cat;
  *|;

  block := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  list := |*
    CRLF list_start { ADD_BLOCK(); LIST_ITEM(); };
    block_end       { ADD_BLOCK(); nest = 0; LIST_ITEM(); fgoto main; };
    default => cat;
  *|;

  main := |*
    notextile_start { ASET(type, notextile); fgoto notextile; };
    bc_start        { ASET(type, bc); fgoto bc; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    list_start      { list_layout = rb_ary_new(); LIST_ITEM(); fgoto list; };
    table           { INLINE(table, table_close); DONE(table); fgoto block; };
    default
    {
      regs = rb_hash_new();
      rb_hash_aset(regs, ID2SYM(rb_intern("type")), plain_block);
      CAT(block);
      fgoto block;
    };
    pre_start       { ASET(type, notextile); CAT(block); fgoto pre; };
    EOF;
  *|;

}%%

%% write data nofinal;

VALUE
superredcloth_transform(rb_formatter, p, pe)
  VALUE rb_formatter;
  char *p, *pe;
{
  int cs, act, nest;
  char *tokstart = NULL, *tokend = NULL, *reg = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = rb_hash_new();
  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE plain_block = rb_str_new2("p");

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    ADD_BLOCK();
  }

  return html;
}

VALUE
superredcloth_transform2(formatter, str)
  VALUE formatter, str;
{
  rb_str_cat2(str, "\n");
  StringValue(str);
  return superredcloth_transform(formatter, RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(super_HTML, self);
}

static VALUE
superredcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(formatter, self);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  rb_define_method(super_RedCloth, "to", superredcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  super_HTML = rb_define_module_under(super_RedCloth, "HTML");
}

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#define superredcloth_scan_c

#include <ruby.h>
#include "superredcloth.h"

VALUE super_ParseError, super_RedCloth, super_HTML;

%%{

  machine superredcloth_scan;
  include superredcloth_common "ext/superredcloth_scan/superredcloth_common.rl";

  action notextile { rb_str_append(html, rb_funcall(rb_formatter, rb_intern("ignore"), 1, regs)); }
  action extend { plain_block = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }
  action no_extend { plain_block = rb_str_new2("p"); }

  # blocks
  notextile_start = "<notextile>" ;
  notextile_end = "</notextile>" ;
  pre_start = "<pre" [^>]* ">" ;
  pre_end = "</pre>" ;
  bc_start = ( "bc" A C :> "." ( "." %extend | "" %no_extend ) " "+ ) ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "bq" | "bc" | "pre" | "notextile" | "div" ) >A %{ STORE(type) } ;
  block_start = ( btype A C :> "." ( "." %extend | "" %no_extend ) " "+ ) ;
  block_end = ( CRLF{2} | EOF );
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  footnote_start = ( ftype A C :> dotspace ) ;
  ul = "*" %{nest++; list_type = "ul";};
  ol = "#" %{nest++; list_type = "ol";};
  list_start  = ( ( ul | ol )+ N A C :> " "+ ) >{nest = 0;} ;

  # tables
  para = ( default+ ) -- CRLF ;
  btext = para ( CRLF{2} )? ;
  tddef = ( S A C D? :> dotspace ) ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, text, td);} ) >X ;
  trdef = ( A C :> dotspace ) ;
  tr = ( trdef? "|" %{INLINE(table, tr_open);} td+ ) >X %{INLINE(table, tr_close);} ;
  trows = ( tr (CRLF >X tr)* ) ;
  tdef = ( "table" >X A C :> dotspace CRLF ) ;
  table = ( tdef? trows >{INLINE(table, table_open);} ) >{ reg = NULL; } ;

  bc := |*
    block_end       { ADD_BLOCKCODE(); fgoto main; };
    default => cat;
  *|;

  pre := |*
    pre_end         { DONE(block); fgoto main; };
    default => cat;
  *|;

  notextile := |*
    notextile_end   { DONE(block); fgoto main; };
    default => cat;
  *|;

  block := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  list := |*
    CRLF list_start { ADD_BLOCK(); LIST_ITEM(); };
    block_end       { ADD_BLOCK(); nest = 0; LIST_ITEM(); fgoto main; };
    default => cat;
  *|;

  main := |*
    notextile_start { ASET(type, notextile); fgoto notextile; };
    bc_start        { ASET(type, bc); fgoto bc; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    list_start      { list_layout = rb_ary_new(); LIST_ITEM(); fgoto list; };
    table           { INLINE(table, table_close); DONE(table); fgoto block; };
    default
    {
      regs = rb_hash_new();
      rb_hash_aset(regs, ID2SYM(rb_intern("type")), plain_block);
      CAT(block);
      fgoto block;
    };
    pre_start       { ASET(type, notextile); CAT(block); fgoto pre; };
    EOF;
  *|;

}%%

%% write data nofinal;

VALUE
superredcloth_transform(rb_formatter, p, pe)
  VALUE rb_formatter;
  char *p, *pe;
{
  int cs, act, nest;
  char *tokstart = NULL, *tokend = NULL, *reg = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = rb_hash_new();
  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE plain_block = rb_str_new2("p");

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    ADD_BLOCK();
  }

  return html;
}

VALUE
superredcloth_transform2(formatter, str)
  VALUE formatter, str;
{
  rb_str_cat2(str, "\n");
  StringValue(str);
  return superredcloth_transform(formatter, RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(super_HTML, self);
}

static VALUE
superredcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(formatter, self);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  rb_define_method(super_RedCloth, "to", superredcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  super_HTML = rb_define_module_under(super_RedCloth, "HTML");
}