Remove unneeded Ragel machine for HTML attrs

We can just use the same machine here as used for XML attribute values.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline = '\r\n' | '\n' | '\r';

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple(id_on_doctype_start);
        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    action hold_start_element_head {
        fhold;
        fnext element_head;
    }

    # Characters that can be used for unquoted HTML attribute values.
    # See https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example
    # for more info.
    html_unquoted_value = ^(
        squote | dquote | '`' | '=' | '<' | '>' | whitespace | newline
    )+;

    # Machine used for processing HTML attribute values.
    html_attribute_value := |*
        squote | dquote => {
            fhold;
            fnext xml_attribute_value;
        };

        # Unquoted attribute values are lexed as if they were single quoted
        # strings.
        html_unquoted_value => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_start_element_head;
    *|;

    # Machine used for processing XML attribute values.
    xml_attribute_value := |*
        squote => start_string_squote;
        dquote => start_string_dquote;
        any    => hold_start_element_head;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace;

        newline => {
            callback_simple(id_advance_line);
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            if ( html_p )
            {
                fnext html_attribute_value;
            }
            else
            {
                fnext xml_attribute_value;
            }
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( literal_html_element_p() )
            {
                fnext literal_html_element;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # Certain tags in HTML can contain basically anything except for the literal
    # closing tag. Two examples are script and style tags.  As a result of this
    # we can't use the regular text machine.
    literal_html_closing_tags = '</script>' | '</style>';
    literal_html_allowed = (any* -- literal_html_closing_tags) $count_newlines;

    literal_html_element := |*
        literal_html_allowed => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        literal_html_allowed %{ mark = p; } literal_html_closing_tags => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment_start  => start_comment;
        cdata_start    => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a ID object, for Java
    # it should be a String. For example:
    #
    #     ID id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline = '\r\n' | '\n' | '\r';

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';

    # Everything except "-" OR a single "-"
    comment_allowed = (^'-'+ | '-') $count_newlines;

    action start_comment {
        callback_simple(id_on_comment_start);

        fnext comment_body;
    }

    comment_body := |*
        comment_allowed => {
            callback(id_on_comment_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        comment_end => {
            callback_simple(id_on_comment_end);

            fnext main;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Everything except "]" OR a single "]"
    cdata_allowed = (^']'+ | ']') $count_newlines;

    action start_cdata {
        callback_simple(id_on_cdata_start);

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_allowed => {
            callback(id_on_cdata_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        cdata_end => {
            callback_simple(id_on_cdata_end);

            fnext main;
        };
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    # Everything except "?" OR a single "?"
    proc_ins_allowed = (^'?'+ | '?') $count_newlines;

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_allowed => {
            callback(id_on_proc_ins_body, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        proc_ins_end => {
            callback_simple(id_on_proc_ins_end);

            fnext main;
        };
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple(id_on_doctype_start);
        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback_simple(id_on_xml_decl_end);

            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any $count_newlines;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    action hold_start_element_head {
        fhold;
        fnext element_head;
    }

    # Characters that can be used for unquoted HTML attribute values.
    # See https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example
    # for more info.
    html_unquoted_value = ^(
        squote | dquote | '`' | '=' | '<' | '>' | whitespace | newline
    )+;

    # Machine used for processing HTML attribute values.
    html_attribute_value := |*
        squote | dquote => {
            fhold;
            fnext html_attribute_value_quoted;
        };

        # Unquoted attribute values are lexed as if they were single quoted
        # strings.
        html_unquoted_value => {
            callback_simple(id_on_string_squote);

            callback(id_on_string_body, data, encoding, ts, te);

            callback_simple(id_on_string_squote);
        };

        any => hold_start_element_head;
    *|;

    # Machine specifically used when dealing with quoted HTML attributes. This
    # ensures that input such as <a href="foo"/> doesn't result in "/" being
    # considered part of the attribute value.
    html_attribute_value_quoted := |*
        squote => start_string_squote;
        dquote => start_string_dquote;
        any    => hold_start_element_head;
    *|;

    # Machine used for processing XML attribute values.
    xml_attribute_value := |*
        squote => start_string_squote;
        dquote => start_string_dquote;
        any    => hold_start_element_head;
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace;

        newline => {
            callback_simple(id_advance_line);
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        '=' => {
            if ( html_p )
            {
                fnext html_attribute_value;
            }
            else
            {
                fnext xml_attribute_value;
            }
        };

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);

            if ( literal_html_element_p() )
            {
                fnext literal_html_element;
            }
            else
            {
                fnext main;
            }
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # Certain tags in HTML can contain basically anything except for the literal
    # closing tag. Two examples are script and style tags.  As a result of this
    # we can't use the regular text machine.
    literal_html_closing_tags = '</script>' | '</style>';
    literal_html_allowed = (any* -- literal_html_closing_tags) $count_newlines;

    literal_html_element := |*
        literal_html_allowed => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        literal_html_allowed %{ mark = p; } literal_html_closing_tags => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment_start  => start_comment;
        cdata_start    => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%


nicer output for return values

nicer output for return values

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ('<' . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


partial work for feature #217 not including option to postprocess warnings.log as this is likely to cause PARSE_ERRORS all the time however, left the option of postprocessing a single file only.

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute' . (' '?)) => {};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&\$:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&\$:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    (('<'|'$<$') . [^&$]* . ('$>$'|'>')) => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>'|'$>$') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    (('<'|'$<$') . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - ('\&'|'\$'))*) => {
      if (*p == '\&')
        fout << " ";
      fout.write(ts+1, te-ts-1);
      if (*p == '\&')
        fout << "";
    };

    # end of return value
    ('>'|'$>$') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  retvals := |*

    ('<' | '$<$') => {};

    ('mlhs' . '\\\-'? . 'Inner' . '\\\-'? . 'Subst' . '\\\*'?) => { fcall retval; };

    ('>' | '$>$') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  # function name
  mtocsubst:= |*
    ('_' | '\\_');

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    ('_m_tsbus_cotm_' )  => { fout << ">"; fgoto main; };

    ( '\\-m\\-\\_\\-tsbus\\-\\_\\-cotm') => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhs' . ('\\-')? . 'Subst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhs' . ('\\-')? . 'Inner' . ('\\-')? . 'Subst' . ('\\\*')?) => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::' . ('\\-')? . 'substitute' . ('\\\*')?) => {fout << "function ";};

   # function name
   ('mtoc_subst_' | 'mtoc\\-\\_\\-subst\\-\\_\\-') => { fgoto mtocsubst; };

   ('::mtoc_subst_' | ('\\-'? . '::' . '\\-' . 'mtoc\\-\\_\\-subst\\-\\_')) => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute'. (' '?)) => {};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ( ('<' '$<$') . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&\$:.,;_\t\-])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&\$\-]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;
  bool         dry_run_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag, const bool dry_run_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag),
    dry_run_(dry_run_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout2;
    if (!dry_run_)
    {
      try {
        fout2.open(file.c_str(), ios_base::trunc);
      } catch (std::ofstream::failure e) {
        cerr << "Exception opening/writing file";
        exit(-1);
      }
    }

    std::ostream * fout_ptr;

    if (dry_run_)
      fout_ptr = &std::cout;
    else
      fout_ptr = &fout2;
    std::ostream & fout = *fout_ptr;

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    if (!dry_run_)
      fout2.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] target" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output.\n" << endl;
  cout << "  -f\t\tsingle file argument instead of directory." << endl;
}

/**
 * @change{dw,1,4,2012-11-19} Added the possibility to directly specify a file target instead of a whole folder.
 */
int main(int argc, char ** argv)
{
  bool quiet = false;
  bool dry_run = false;
  bool isfile = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-d") == std::string(argv[1]))
    {
      dry_run = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-f") == std::string(argv[1]))
	{
	  isfile = true;
	  docdir = argv[2];
	}
    else if (argc == 3 && (std::string("-qf") == std::string(argv[1]) || std::string("-fq") == std::string(argv[1])))
	{
	  isfile = true;
	  quiet = true;
	  docdir = argv[2];
	}
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet) {
	  if (isfile) {
		  cout << "Running mtoc++ postprocessor on file " << docdir << endl;
	  } else {
		  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;
	  }
  }

  PostProcess scanner(docdir, quiet, dry_run);
  if (isfile) {
	  scanner.postprocess(docdir);
  } else {
	  scanner.execute();
  }
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute' . (' '?)) => {};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&\$:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&\$:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    (('<'|'$<$') . [^&$]* . ('$>$'|'>')) => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>'|'$>$') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    (('<'|'$<$') . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - ('\&'|'\$'))*) => {
      if (*p == '\&')
        fout << " ";
      fout.write(ts+1, te-ts-1);
      if (*p == '\&')
        fout << "";
    };

    # end of return value
    ('>'|'$>$') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  retvals := |*

    ('<' | '$<$') => {};

    ('mlhs' . '\\\-'? . 'Inner' . '\\\-'? . 'Subst' . '\\\*'?) => { fcall retval; };

    ('>' | '$>$') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  # function name
  mtocsubst:= |*
    ('_' | '\\_');

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    ('_m_tsbus_cotm_' )  => { fout << ">"; fgoto main; };

    ( '\\-m\\-\\_\\-tsbus\\-\\_\\-cotm') => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhs' . ('\\-')? . 'Subst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhs' . ('\\-')? . 'Inner' . ('\\-')? . 'Subst' . ('\\\*')?) => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::' . ('\\-')? . 'substitute' . ('\\\*')?) => {fout << "function ";};

   # function name
   ('mtoc_subst_' | 'mtoc\\-\\_\\-subst\\-\\_\\-') => { fgoto mtocsubst; };

   ('::mtoc_subst_' | ('\\-'? . '::' . '\\-' . 'mtoc\\-\\_\\-subst\\-\\_')) => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute'. (' '?)) => {};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ( ('<' '$<$') . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&\$:.,;_\t\-])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&\$\-]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;
  bool         dry_run_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag, const bool dry_run_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag),
    dry_run_(dry_run_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout2;
    if (!dry_run_)
    {
      try {
        fout2.open(file.c_str(), ios_base::trunc);
      } catch (std::ofstream::failure e) {
        cerr << "Exception opening/writing file";
        exit(-1);
      }
    }

    std::ostream * fout_ptr;

    if (dry_run_)
      fout_ptr = &std::cout;
    else
      fout_ptr = &fout2;
    std::ostream & fout = *fout_ptr;

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    if (!dry_run_)
      fout2.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] filename" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output." << endl;
}

int main(int argc, char ** argv)
{
  bool quiet = false;
  bool dry_run = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-d") == std::string(argv[1]))
    {
      dry_run = true;
      docdir = argv[2];
    }
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet)
    cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir, quiet, dry_run);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


Ragel: refactoring

Ragel: refactoring

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call(:string,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call(:integer,data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call(:float,data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call(:identifier,data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call(:bool,data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  values.each do | v |
    p v
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  p s
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call(:string,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call(:integer,data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call(:float,data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call(:identifier,data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call(:bool,data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  p s
  VcfHeader::RagelKeyValues.run_lexer(s, debug: true)
}

end


add more tests - comments to tests

add more tests - comments to tests

#include "chunked_message.h"
#include <stdio.h>
#include <assert.h>

#ifndef MIN
# define MIN(a,b) (a < b ? a : b)
#endif

#define LEN (p - parser->mark)
#define REMAINING (pe - p)
%%{
  machine chunked_parser;

  action mark { parser->mark = p; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;

    if( 'A' <= *p && *p <= 'F')
      parser->chunk_size += *p - 'A' + 10;
    else if( 'a' <= *p && *p <= 'f')
      parser->chunk_size += *p - 'a' + 10;
    else if( '0' <= *p && *p <= '9')
      parser->chunk_size += *p - '0';
    else
      assert(0 && "bad hex char");

  }

  action start_field {;}
  action write_field {;}
  action start_value {;}
  action write_value {;}

  action skip_data {
    //printf("chunk_size: %d\n", parser->chunk_size);
    if(parser->chunk_size > REMAINING) {
      parser->eating = 1;
      parser->chunk_handler(parser->data, p, REMAINING);
      parser->chunk_size -= REMAINING;
      fhold;
      fbreak;
    } else {
      parser->chunk_handler(parser->data, p, parser->chunk_size);
      p += parser->chunk_size;
      parser->chunk_size = 0;
      parser->eating = 0;
      fhold;
      fgoto chunk_end;
    }
  }

  include http11 "http11.rl";

  trailer = message_header*;

  nonzero_xdigit = [1-9a-fA-F];
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val   = token*;

  chunk_ext_name  = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* nonzero_xdigit xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_data;
  chunk_begin  = chunk_size chunk_extension CRLF;
  chunk        = chunk_begin chunk_body chunk_end;
  Chunked_Body = chunk* last_chunk trailer CRLF;

main := Chunked_Body;

}%%

%% write data;

void chunked_parser_init
  ( chunked_parser *parser
  )
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->chunk_size = 0;
  parser->eating = 0;

  parser->mark = NULL;
  parser->nread = 0;
  parser->chunk_handler = NULL;
}

/** exec **/
size_t chunked_parser_execute
  ( chunked_parser *parser
  , const char *buffer
  , size_t len
  )
{
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if(0 < parser->chunk_size && parser->eating) {
    size_t eat = MIN(len, parser->chunk_size);
    if(eat == parser->chunk_size) {
      parser->eating = 0;
    }
    parser->chunk_handler(parser->data, p, eat);
    p += eat;
    parser->chunk_size -= eat;
    //printf("eat: %d\n", eat);
  }

  %% write exec;

  parser->cs = cs;
  parser->nread += p - buffer;

  assert(p <= pe && "buffer overflow after parsing execute");

  if(parser->mark)
    assert(parser->mark < pe && "mark is after buffer end");

  return(p - buffer);
}

int chunked_parser_has_error
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_error;
}

int chunked_parser_is_finished
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_first_final;
}

#ifdef UNITTEST
#include <string.h>

static chunked_parser parser;
static char test_buf[200];

void chunk_handler(void *data, const char *p, size_t len)
{
  //printf("chunk_handler: '%s'", test_buf);
  strncat(test_buf, p, len);
  //printf(" -> '%s'\n", test_buf);
}

int test_string(const char *buf)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf, strlen(buf));

  if(!chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  return 1;
}

int test_split(const char *buf1, const char *buf2)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf1, strlen(buf1));

  //printf("test_buf: %s\n", test_buf);

  if(chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  traversed += chunked_parser_execute(&parser, buf2, strlen(buf2));

  if(chunked_parser_has_error(&parser))
    return -3;
  if(!chunked_parser_is_finished(&parser))
    return -4;

  //printf("test_buf: %s\n", test_buf);

  return 1;
}

int main()
{
  // zero length
  assert(0 < test_string("0\r\n\r\n"));
  assert(0 == strcmp("", test_buf));
  assert(0 < test_string("00000\r\n\r\n"));
  assert(0 == strcmp("", test_buf));

  // easy - one chunk
  assert(0 < test_string("5\r\nhello\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello", test_buf));

  // two chunks ; triple zero ending
  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n000\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with trailing headers. blech.
  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nContent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with bullshit after the length
  assert(0 < test_string("5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken between \r and \n between chunks
  assert(0 < test_split("5\r\nhello\r", "\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken at start of chunk
  assert(0 < test_split("5\r\nhello\r\n", "6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken at end of chunk
  assert(0 < test_split("5\r\nhello", "\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken in middle of chunk
  assert(0 < test_split("5\r\nhel", "lo\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken in just after the zero
  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0", "\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken one char before the end
  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0\r\n\r", "\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken in trailing headers
  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nCon", "tent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken in chunk_extension
  assert(0 < test_split("5; iha", "tew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // broken in chunk_extension just before = sign
  assert(0 < test_split("5; ihatew3;whatthefuck", "=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // now work with "all your base are belong to us"
  // because it is two digits in length (0x1e = 30)

  // easy
  assert(0 < test_string("1e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  // broken mid-chunk
  assert(0 < test_split("1e\r\nall your", " base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  // broken between first CRLF
  assert(0 < test_split("1e\r", "\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  // broken on chunk length
  assert(0 < test_split("1", "e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  printf("okay\n");
  return 0;
}

#endif

#include "chunked_message.h"
#include <stdio.h>
#include <assert.h>

#ifndef MIN
# define MIN(a,b) (a < b ? a : b)
#endif

#define LEN (p - parser->mark)
#define REMAINING (pe - p)
%%{
  machine chunked_parser;

  action mark { parser->mark = p; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;

    if( 'A' <= *p && *p <= 'F')
      parser->chunk_size += *p - 'A' + 10;
    else if( 'a' <= *p && *p <= 'f')
      parser->chunk_size += *p - 'a' + 10;
    else if( '0' <= *p && *p <= '9')
      parser->chunk_size += *p - '0';
    else
      assert(0 && "bad hex char");

  }

  action start_field {;}
  action write_field {;}
  action start_value {;}
  action write_value {;}

  action skip_data {
    //printf("chunk_size: %d\n", parser->chunk_size);
    if(parser->chunk_size > REMAINING) {
      parser->eating = 1;
      parser->chunk_handler(parser->data, p, REMAINING);
      parser->chunk_size -= REMAINING;
      fhold;
      fbreak;
    } else {
      parser->chunk_handler(parser->data, p, parser->chunk_size);
      p += parser->chunk_size;
      parser->chunk_size = 0;
      parser->eating = 0;
      fhold;
      fgoto chunk_end;
    }
  }

  include http11 "http11.rl";

  trailer = message_header*;

  nonzero_xdigit = [1-9a-fA-F];
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val   = token*;

  chunk_ext_name  = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* nonzero_xdigit xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_data;
  chunk_begin  = chunk_size chunk_extension CRLF;
  chunk        = chunk_begin chunk_body chunk_end;
  Chunked_Body = chunk* last_chunk trailer CRLF;

main := Chunked_Body;

}%%

%% write data;

void chunked_parser_init
  ( chunked_parser *parser
  )
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->chunk_size = 0;
  parser->eating = 0;

  parser->mark = NULL;
  parser->nread = 0;
  parser->chunk_handler = NULL;
}

/** exec **/
size_t chunked_parser_execute
  ( chunked_parser *parser
  , const char *buffer
  , size_t len
  )
{
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if(0 < parser->chunk_size && parser->eating) {
    size_t eat = MIN(len, parser->chunk_size);
    if(eat == parser->chunk_size) {
      parser->eating = 0;
    }
    parser->chunk_handler(parser->data, p, eat);
    p += eat;
    parser->chunk_size -= eat;
    //printf("eat: %d\n", eat);
  }

  %% write exec;

  parser->cs = cs;
  parser->nread += p - buffer;

  assert(p <= pe && "buffer overflow after parsing execute");

  if(parser->mark)
    assert(parser->mark < pe && "mark is after buffer end");

  return(p - buffer);
}

int chunked_parser_has_error
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_error;
}

int chunked_parser_is_finished
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_first_final;
}

#ifdef UNITTEST
#include <string.h>

static chunked_parser parser;
static char test_buf[200];

void chunk_handler(void *data, const char *p, size_t len)
{
  //printf("chunk_handler: '%s'", test_buf);
  strncat(test_buf, p, len);
  //printf(" -> '%s'\n", test_buf);
}

int test_string(const char *buf)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf, strlen(buf));

  if(!chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  return 1;
}

int test_split(const char *buf1, const char *buf2)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf1, strlen(buf1));

  //printf("test_buf: %s\n", test_buf);

  if(chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  traversed += chunked_parser_execute(&parser, buf2, strlen(buf2));

  if(chunked_parser_has_error(&parser))
    return -3;
  if(!chunked_parser_is_finished(&parser))
    return -4;

  //printf("test_buf: %s\n", test_buf);

  return 1;
}

int main()
{
  assert(0 < test_string("5\r\nhello\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello", test_buf));

  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with trailing headers. blech.
  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nContent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with bullshit after the length
  assert(0 < test_string("5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r", "\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r\n", "6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello", "\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhel", "lo\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0", "\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0\r\n\r", "\n"));
  assert(0 == strcmp("hello world", test_buf));

  // split with trailing headers. blech.
  assert(0 < test_split("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nCon", "tent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // split with bullshit after the length
  assert(0 < test_split("5; iha", "tew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // now work with "all your base are belong to us"
  // because it is two digits in length (0x1e = 30)

  assert(0 < test_string("1e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r\nall your", " base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r", "\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1", "e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  printf("okay\n");
  return 0;
}

#endif


fixup string machine mess

fixup string machine mess

I mixed actions with definitions. Oops. Also @'d when I should have %'d,
and that's always a problem. Don't do that.

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_string {
    #$stderr.puts 'start_string'
    @incomplete[:string] = []
    @opened = true
  }

  action accum_string {
    #$stderr.puts 'accum_string "' + fc.chr + '"'
    @incomplete[:string] << fc
  }

  action end_string {
    #$stderr.puts 'end_string'
    @closed = true
  }

  action string_end {
    #$stderr.puts 'string_end'
    string = @incomplete.delete(:string) || []
    completed = @opened && @closed
    ast_node = string(utf8_string(string), complete: completed)
    @buffers[:string] = ast_node
  }

  action string_yield {
    yield @buffers[:string]
  }

  action eof_string {
    #$stderr.puts 'eof_string'
    unless @closed
      $stderr.puts "incomplete string - why?"
    else
      $stderr.puts "complete string"
    end
  }

  NOT_DQ_ESC = [^"\\];
  string_prefix = DQ %start_string;
  string_content = ( NOT_DQ_ESC | ESCAPED )* $accum_string;
  string_suffix = DQ %end_string;
  str_string = string_prefix string_content string_suffix %string_end;
  str_ast := (string_prefix? |
              string_prefix string_content |
              string_prefix string_content string_suffix)
              NL?
              @string_end
              @string_yield
              $eof(eof_string);
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module String

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @opened     = false
            @closed     = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::String.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action string_start {
    @incomplete[:string] = []
    @opened = true
  }

  action string_more {
    @incomplete[:string] << fc
  }

  action string_finish {
    @closed = true
  }

  action string_end {
    string = @incomplete.delete(:string) || []
    completed = @opened && @closed
    ast_node = string(utf8_string(string), complete: completed)
    @buffers[:string] = ast_node
  }

  action string_yield {
    yield @buffers[:string]
  }

  action eof_string {
    $stderr.puts 'eof_string'
    unless @closed
      $stderr.puts "incomplete string - why?"
    else
      $stderr.puts "complete string"
    end
  }

  NOT_DQ_ESC = [^"\\];
  start_string = DQ %string_start;
  more_string = ( NOT_DQ_ESC | ESCAPED )* $string_more;
  end_string = DQ %string_finish;
  str_string = start_string more_string end_string @string_end;
  str_ast := (start_string? |
              start_string more_string |
              start_string more_string end_string)
              NL?
              @string_end
              @string_yield
              $eof(eof_string);
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module String

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @opened     = false
            @closed     = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::String.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Ragel handle negative numbers

Ragel handle negative numbers

#include "../common.h"
// Ragel version of issuu_json2sql

int echo = 0;

%%{

  machine issuu_json2sql;

  action echo_on  { echo = 1; }
  action echo_off { echo = 0; }
  action print {
    if(echo) {
      fprintf(stdout, "%c", fc);
    } else {
      // fprintf(stdout, "[%c]", fc);
    }
  }

  ws = [ \n]*;
  sep = ws ':' ws;
  keepComma = ws ',' %{ fputs(",", stdout); };
  dropComma = ws ',';
  someInt = /-?[0-9]*/;
  someString = /[^"\n]*/;
  qt = '\"' > { fputs("'", stdout); };
  stringReplaceQuotes = qt (someString > echo_on % echo_off) qt;
  fb = '\"' someString '\"' sep ( '\"' someString '\"'
                                | someInt
                                ) dropComma?;
  keyVal = ( '\"ts\"'                sep (someInt > echo_on % echo_off) keepComma
           | '\"visitor_uuid\"'      sep stringReplaceQuotes keepComma
           | '\"visitor_useragent\"' sep stringReplaceQuotes keepComma
           | '\"visitor_country\"'   sep stringReplaceQuotes dropComma
           | fb
           );
  curl_start = '{' > { fputs("(", stdout); };
  curl_end   = '}' > { fputs(")", stdout); };
  object = curl_start (ws keyVal)* ws curl_end;

  json2sql = (object ws)*;

  main := json2sql $ print;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  fputs("INSERT INTO issuu_log (ts, visitor_uuid, visitor_useragent, visitor_country) VALUES\n", stdout);

  int first = 1;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    char *p  = &buffer[0];
    char *pe = p + strlen(buffer) + 1;
    if(!first) {
      fputs(",\n", stdout);
    }
    first = 0;
    %% write init;
    %% write exec;
  }
  fputs(";\n", stdout);

  POST;

  return 0;
}

#include "../common.h"
// Ragel version of issuu_json2sql

int echo = 0;

%%{

  machine issuu_json2sql;

  action echo_on  { echo = 1; }
  action echo_off { echo = 0; }
  action print {
    if(echo) {
      fprintf(stdout, "%c", fc);
    } else {
      // fprintf(stdout, "[%c]", fc);
    }
  }

  ws = [ \n]*;
  sep = ws ':' ws;
  keepComma = ws ',' %{ fputs(",", stdout); };
  dropComma = ws ',';
  someInt = /[0-9]*/;
  someString = /[^"\n]*/;
  qt = '\"' > { fputs("'", stdout); };
  stringReplaceQuotes = qt (someString > echo_on % echo_off) qt;
  fb = '\"' someString '\"' sep ( '\"' someString '\"'
                                | someInt
                                ) dropComma?;
  keyVal = ( '\"ts\"'                sep (someInt > echo_on % echo_off) keepComma
           | '\"visitor_uuid\"'      sep stringReplaceQuotes keepComma
           | '\"visitor_useragent\"' sep stringReplaceQuotes keepComma
           | '\"visitor_country\"'   sep stringReplaceQuotes dropComma
           | fb
           );
  curl_start = '{' > { fputs("(", stdout); };
  curl_end   = '}' > { fputs(")", stdout); };
  object = curl_start (ws keyVal)* ws curl_end;

  json2sql = (object ws)*;

  main := json2sql $ print;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  fputs("INSERT INTO issuu_log (ts, visitor_uuid, visitor_useragent, visitor_country) VALUES\n", stdout);

  int first = 1;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    char *p  = &buffer[0];
    char *pe = p + strlen(buffer) + 1;
    if(!first) {
      fputs(",\n", stdout);
    }
    first = 0;
    %% write init;
    %% write exec;
  }
  fputs(";\n", stdout);

  POST;

  return 0;
}


redoing string.rl; WIP

redoing string.rl; WIP

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start {
    $stderr.puts '<< start'
    @incomplete[:string] = []
  }

  action append {
    $stderr.puts '<< append'
    @incomplete[:string] << fc
  }

  action end {
    $stderr.puts '<< end'
    string = @incomplete.delete(:string)
    completed = !string.empty?
    ast_node = string(utf8_string(string), complete: completed)
    @buffers[:string] = ast_node
  }

  action yield {
    $stderr.puts '<< yield'
    yield @buffers[:string]
  }

  action ast_end {
    $stderr.puts '<< end1'
    unless @buffers.key?(:string)
      $stderr.puts 'ast_end'
      string = @incomplete.delete(:string) || []
      completed = !string.empty?
      ast_node = string(utf8_string(string), complete: completed)
      @buffers[:string] = ast_node
    end
  }

  STR_CHARS = ('"' ('\\\"' | [^"])** '"');
  STRING = STR_CHARS >start $append %end;
  PARTIAL = STR_CHARS >start $append;
  AST_NODE := (STRING | PARTIAL) NL? %/ast_end %yield;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module String

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @ended      = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::String.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_string {
    @incomplete[:string] = []
  }

  action append_string {
    @incomplete[:string] << fc
  }

  action end_string {
    string = @incomplete.delete(:string)
    completed = !string.empty?
    ast_node = string(utf8_string(string), complete: completed)
    @buffers[:string] = ast_node
    @ended = true
  }

  action yield_string {
    yield @buffers[:string]
  }

  action eof_string {
    string = @incomplete.delete(:string)
    completed = !string.empty? && @ended
    ast_node = string(utf8_string(string), complete: completed)
    yield ast_node
  }

  action err_string {
    string = @incomplete.delete(:string) || []
    completed = !string.empty?
    ast_node = string(utf8_string(@buffers[:string]).sub(/\n$/, ''))
    yield ast_node
  }

  STRING =  (
              ('"' ('\\\"' | [^"])** '"') >start_string $append_string %end_string %yield_string |
              '' >start_string %end_string %yield_string
            )
            $eof(eof_string) $err(err_string);

  string :=
    STRING NL;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module String

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @ended      = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::String.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


[raytrace] Deleted an unneeded output file from raytrace