Docs for the shared Ragel grammar.

Docs for the shared Ragel grammar.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_:]+;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    # Machine for processing double quoted strings.
    string_dquote := |*
        ^dquote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        dquote => { fret; };
    *|;

    # Machine for processing single quoted strings.
    string_squote := |*
        ^squote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        squote => { fret; };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Lex the public/system IDs as regular strings.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # CDATA tags are broken up into 3 parts: the start, the content and the
    # end tag.
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #
    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    action start_cdata {
        callback_simple("on_cdata_start");
        fcall cdata;
    }

    # Machine that for processing the contents of CDATA tags. Everything
    # inside a CDATA tag is treated as plain text.
    cdata := |*
        any* cdata_end => {
            callback("on_text", data, encoding, ts, te - 3);
            callback_simple("on_cdata_end");
            fret;
        };
    *|;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Comments are lexed into 3 parts: the start tag, the content and the
    # end tag.
    #
    # Unlike the W3 specification these rules *do* allow character
    # sequences such as `--` and `->`. Putting extra checks in for these
    # sequences would actually make the rules/actions more complex.
    #
    comment_start = '<!--';
    comment_end   = '-->';

    action start_comment {
        callback_simple("on_comment_start");
        fcall comment;
    }

    # Machine used for processing the contents of a comment. Everything
    # inside a comment is treated as plain text (similar to CDATA tags).
    comment := |*
        any* comment_end => {
            callback("on_text", data, encoding, ts, te - 3);
            callback_simple("on_comment_end");
            fret;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        callback("on_element_start", data, encoding, ts + 1, te);

        fcall element_head;
    }

    element_start = '<' identifier;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        element_start  => start_element;
        doctype_start  => start_doctype;
        cdata_start    => start_cdata;
        comment_start  => start_comment;
        xml_decl_start => start_xml_decl;

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%

%%machine base_lexer;

%%{
    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_:]+;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    # Machine for processing double quoted strings.
    string_dquote := |*
        ^dquote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        dquote => { fret; };
    *|;

    # Machine for processing single quoted strings.
    string_squote := |*
        ^squote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        squote => { fret; };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Lex the public/system IDs as regular strings.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # CDATA tags are broken up into 3 parts: the start, the content and the
    # end tag.
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #
    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    action start_cdata {
        callback_simple("on_cdata_start");
        fcall cdata;
    }

    # Machine that for processing the contents of CDATA tags. Everything
    # inside a CDATA tag is treated as plain text.
    cdata := |*
        any* cdata_end => {
            callback("on_text", data, encoding, ts, te - 3);
            callback_simple("on_cdata_end");
            fret;
        };
    *|;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Comments are lexed into 3 parts: the start tag, the content and the
    # end tag.
    #
    # Unlike the W3 specification these rules *do* allow character
    # sequences such as `--` and `->`. Putting extra checks in for these
    # sequences would actually make the rules/actions more complex.
    #
    comment_start = '<!--';
    comment_end   = '-->';

    action start_comment {
        callback_simple("on_comment_start");
        fcall comment;
    }

    # Machine used for processing the contents of a comment. Everything
    # inside a comment is treated as plain text (similar to CDATA tags).
    comment := |*
        any* comment_end => {
            callback("on_text", data, encoding, ts, te - 3);
            callback_simple("on_comment_end");
            fret;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        callback("on_element_start", data, encoding, ts + 1, te);

        fcall element_head;
    }

    element_start = '<' identifier;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        element_start  => start_element;
        doctype_start  => start_doctype;
        cdata_start    => start_cdata;
        comment_start  => start_comment;
        xml_decl_start => start_xml_decl;

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%


Add more fields to the scanner

Add more fields to the scanner

package handbrake

import (
	"fmt"
)

%%{
	machine handbrake;
	write data;
}%%

func parseOutput(data string) (HandBrakeMeta) {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	_,_,_,_ = top, ts, te, act
	stack := []int{0}
	_ = eof
	line := 1
	capture := false
	csp := 0
	meta := HandBrakeMeta{}
	fmt.Printf("%02d: ", line)
	_ = capture
	_ = csp
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		title := |*
			(alnum|space)+[.]*alnum* => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		duration := |*
			space*;
			digit{2}[:]digit{2}[:]digit{2} => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} => { fmt.Printf("%s", data[ts:te]); fret;};
		*|;
		paspect := |*
			space*;
			digit{1,4} "/" digit{1,4} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		daspect := |*
			space*;
			digit{1} . "." . digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		fps := |*
			"\n" => { ts=ts-12; fmt.Printf("%s\n", data[ts:te-4]); fret; };
		*|;
		crop := |*
			space*;
			digit{1,3} "/" digit{1,3} "/" digit{1,3} "/" digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		main := (
			newline |
			space+ "+" space+ "stream:" space* @{ fcall title; } |
			space+ "+" space+ "duration:" @{ fcall duration; } |
			space+ "+" space+ "size:" @{ fcall picture; } |
			space+ "+" any+ "pixel" space+ "aspect:" @{ fcall paspect; } |
			space+ "+" any+ "display" space+ "aspect:" @{ fcall daspect; } |
			space+ "+" any+ "fps" @{ fcall fps; } |
			space+ "+" space+ "autocrop:" @{ fcall crop; }
		)*;
		write init;
		write exec;
	}%%
	return meta
}

package handbrake

import (
	"fmt"
)

%%{
	machine handbrake;
	write data;
}%%

func parseOutput(data string) (HandBrakeMeta) {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	_,_,_,_ = top, ts, te, act
	stack := []int{0}
	_ = eof
	line := 1
	capture := false
	csp := 0
	meta := HandBrakeMeta{}
	fmt.Printf("%02d: ", line)
	_ = capture
	_ = csp
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		title := |*
			(alnum|space)+[.]*alnum* => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		duration := |*
			space*;
			digit{2}[:]digit{2}[:]digit{2} => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} => { fmt.Printf("%s", data[ts:te]); fret;};
		*|;
		main := (
			newline |
			space+ "+" space+ "stream:" space* @{ fcall title; } |
			space+ "+" space+ "duration:" @{ fcall duration; } |
			space+ "+" space+ "size:" @{ fcall picture; }
		)*;
		write init;
		write exec;
	}%%
	return meta
}


Add missing comma

Add missing comma

#include "toml.h"
#include "toml_private.h"

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>
#include <math.h>
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

bool add_node_to_tree(struct list_head* list_stack, struct toml_node* cur_table, struct toml_node* node, char* name, char** parse_error, int* malloc_error, int cur_line)
{
	struct toml_stack_item *cur_list =
					list_tail(list_stack, struct toml_stack_item, list);

	if (cur_list) {
		if (cur_list->list_type && cur_list->list_type != node->type) {
			asprintf(parse_error,
					"incompatible types list %s this %s line %d\n",
					toml_type_to_str(cur_list->list_type),
					toml_type_to_str(node->type), cur_line);
			return false;
		}
		cur_list->list_type = node->type;

		struct toml_list_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}

		memcpy(&item->node, node, sizeof(*node));
		item->node.name = NULL;
		list_add_tail(&cur_list->node->value.list, &item->list);
	} else {
		struct toml_table_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}
		memcpy(&item->node, node, sizeof(*node));
		item->node.name = name;
		list_add_tail(&cur_table->value.map, &item->map);
	}

	return true;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - ('#'|'='|'"'|whitespace))+					>{ts = p;};
	name_in_double_quotes = (print - '"')+						>{ts = p;};
	name_in_single_quotes = (print - "'")+						>{ts = p;};
	tablename =  (print - ('#'|']'|'['|'"'|whitespace))+		>{ts = p;};
	tablename_in_double_quotes =  (print - '"')+				>{ts = p;};
	tablename_in_single_quotes =  (print - "'")+				>{ts = p;};

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen + 1) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, cur_line);
			fbreak;
		}

		while (ts[namelen] == ' ' || ts[namelen] == '\t')
			namelen--;

		name = strndup(ts, namelen + 1);
	}

	action saw_bool {
		struct toml_node node;

		node.type = TOML_BOOLEAN;
		node.value.integer = number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_int {
		char*				te = p;
		struct toml_node	node;

		node.type = TOML_INT;
		node.value.integer = negative ? -number : number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_float {
		char*				te = p;
		struct toml_node	node;

		floating = strtod(ts, &te);

		node.type = TOML_FLOAT;
		node.value.floating.value = floating;
		node.value.floating.precision = precision;

		if (!node.value.floating.precision && !exponent) {
			asprintf(&parse_error, "bad float\n");
			fbreak;
		}

		exponent = false;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_string {
		int					len = strp - string + 1;
		struct toml_node	node;

		*strp = 0;

		node.type = TOML_STRING;
		node.value.string = malloc(len);
		if (!node.value.string) {
			malloc_error = 1;
			fbreak;
		}
		memcpy(node.value.string, string, len);

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_date {
		char*	te = p;
		struct	toml_node node;

		node.type = TOML_DATE;
		node.value.rfc3339_time.epoch = timegm(&tm);
		node.value.rfc3339_time.offset_sign_negative = time_offset_is_negative;
		node.value.rfc3339_time.offset = time_offset;
		node.value.rfc3339_time.offset_is_zulu = time_offset_is_zulu;
		if (secfrac_ptr)
			node.value.rfc3339_time.sec_frac = strtol(secfrac_ptr, &te, 10);
		else
			node.value.rfc3339_time.sec_frac = -1;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), cur_line);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_inline_table {
		char*					tablename = name;
		struct toml_table_item*	item;
		struct toml_node*		place = cur_table ? cur_table : toml_root;
		bool					found = false;

		inline_table = true;

		list_for_each(&place->value.map, item, map) {
			if (strcmp(item->node.name, tablename) != 0)
				continue;

			found = true;
			break;
		}

		if (found)
		{
			asprintf(&parse_error, "duplicate entry %s\n", tablename);
			fbreak;
		}

		item = malloc(sizeof(*item));
		if (!item) {
			malloc_error = 1;
			fbreak;
		}

		item->node.name = strdup(tablename);
		item->node.type = TOML_TABLE;
		list_head_init(&item->node.value.map);
		list_add_tail(&place->value.map, &item->map);

		cur_table = &item->node;

		free(tablename);
	}

	action saw_table {
		char *ancestor, *tofree = NULL, *tablename;
		int item_added = 0;
		int len = (int)(p-ts);

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, len);

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", len, ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s", len, ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		int		ret;
		char*	tableName;

		tableName = strndup(ts, (int)(p-ts-1));
		ret = SawTableArray(toml_root, tableName, &cur_table, &parse_error);
		free(tableName);
		if (ret)
			fbreak;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	action bad_escape {
		asprintf(&parse_error, "bad escape \\%c", *p);
		fbreak;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			'#'			>{in_text = 0; fcall comment;}				->start	|
			[\t ]+		>{in_text = 0; indent = 0;} @{indent++;}	->start	|
			[\n]		>{in_text = 0; cur_line++;}					->start	|
			[\0]		>{in_text = 0; fbreak;}						->start	|
			[^\t \n]	@{fhold;} %{in_text = 1;}					->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] ${cur_line++; fret;} ),

		# a table
		table: (
			tablename ']' @saw_table										->start	|
			'"' tablename_in_double_quotes '"' @saw_table ']'				->start	|
			"'" tablename_in_single_quotes "'" @saw_table ']'				->start	|
			'[' tablename ']' ']' @saw_table_array							->start	|
			'[' '"' tablename_in_double_quotes '"' ']' @saw_table_array ']'	->start	|
			'[' "'" tablename_in_single_quotes "'" ']' @saw_table_array ']'	->start
		),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]				-> basic_empty_or_multi_line	|
			[^"] ${fhold;}	-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]							-> basic_multi_line_start	|
			[^"] $saw_string ${fhold;}	-> start
		),

		basic_multi_line_start: (
			'\n' ${cur_line++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${cur_line++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${cur_line++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${cur_line++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${cur_line++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'"'	${*strp++='"';}			${fret;}	|
			'/'	${*strp++='/';}			${fret;}	|
			'\\'	${*strp++='\\';}	${fret;}	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0uU"/\\] $bad_escape
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			'_' ? digit ${number *= 10; number += fc-'0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}				->date				|
			[eE]											->exponent_part		|
			[.]	>{precision = 0;}							->fractional_part	|
			[\t ,\]\n\0] $saw_int ${fhold;}					->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9]	${precision++;}				->fractional_part	|
			[eE]								->exponent_part		|
			[^0-9eE]	$saw_float ${fhold;}	->start
		),

		exponent_part: (
			[\-\+0-9]	${exponent=true;}		->exponent_part	|
			[^\-\+0-9]	$saw_float ${fhold;}	->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: ( '' >{secfrac_ptr = NULL; time_offset = 0; time_offset_is_negative = 0; time_offset_is_zulu = 0;}
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);} -> fractional_second_or_offset
		),

		fractional_second_or_offset: (
			'.' digit* >{secfrac_ptr=p;} >{fhold;}	-> time_offset	|
			[^.] @{fhold;}							-> time_offset
		),

		time_offset: (
			('-' @{time_offset_is_negative=1;}|'+')
				digit{2} @{time_offset = atoi(fpc-1) * 60;}
				':'
				digit{2} @{time_offset += atoi(fpc-1);}
				@saw_date								-> start |
			'Z' >{time_offset_is_zulu = 1;} @saw_date	-> start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${cur_line++;}	-> literal_multi_line	|
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${cur_line++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string							-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}	-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'		@{number = 1;}									-> true				|
			'false'		@{number = 0;}									-> false			|
			'"'			${strp = string;}								-> basic_string		|
			[']			${strp = string;}								-> literal_string	|
			('-'|'+')	${negative = fc == '-'; number = 0; ts = p;}	-> number_or_date	|
			digit		${negative = false; ts = p; number = fc-'0';}	-> number_or_date
		),

		# A list of values
		list: (
			'#'		>{fcall comment;}	->list	|
			'\n'	${cur_line++;}		->list	|
			[\t ]						->list	|
			','							->list	|
			']'	$end_list				->start	|
			[^#\t, \n\]] ${fhold;}		->val
		),

		inline_table: (
			','								-> inline_table |
			[\t ]							-> inline_table |
			'}'	@{inline_table = false;}	-> start		|
			[^\t ,}] @{fhold;}				-> key
		),

		# A val can be either a list or a singular value
		val: (
			'#'	>{fcall comment;}	->val			|
			'\n' ${ cur_line++; }	->val			|
			[\t ]					->val			|
			'['	$start_list			->list			|
			']'	$end_list			->start			|
			'{'	$saw_inline_table	->inline_table	|
			[^#\t \n[\]{] ${fhold;}	->singular
		),

		# A regular key
		key: (
			name @{namelen = (int)(p-ts);} whitespace '=' $saw_key								->val	|
			'"' name_in_double_quotes '"' @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val	|
			"'" name_in_single_quotes "'" @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'	>{fcall comment;}	->text	|
			'['						->table	|
			[\t ]					->text	|
			'\n' ${cur_line++;}		->start	|
			[^#[\t \n]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node* toml_root, char* buf, int buflen)
{
	int indent = 0, cs, cur_line = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int precision;
	int namelen;
	int64_t number;
	bool negative;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];
	int in_text = 0;
	int time_offset = 0;
	char* secfrac_ptr;
	bool time_offset_is_negative = 0;
	bool time_offset_is_zulu = 0;
	bool inline_table = false;
	bool exponent = false;

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen + 1;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", cur_line);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, cur_line, p);
		free(parse_error);
		return 1;
	}

	if (in_text) {
		fprintf(stderr, "not in start, line %d\n", cur_line);
		return 1;
	}

	/* check we have consumed the entire buffer */
	if (p != pe) {
		fprintf(stderr, "entire buffer unconsumed, line %d\n", cur_line);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", cur_line, p);
		return 1;
	}

	return 0;
}

#include "toml.h"
#include "toml_private.h"

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>
#include <math.h>
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

bool add_node_to_tree(struct list_head* list_stack, struct toml_node* cur_table, struct toml_node* node, char* name, char** parse_error, int* malloc_error, int cur_line)
{
	struct toml_stack_item *cur_list =
					list_tail(list_stack, struct toml_stack_item, list);

	if (cur_list) {
		if (cur_list->list_type && cur_list->list_type != node->type) {
			asprintf(parse_error,
					"incompatible types list %s this %s line %d\n",
					toml_type_to_str(cur_list->list_type),
					toml_type_to_str(node->type), cur_line);
			return false;
		}
		cur_list->list_type = node->type;

		struct toml_list_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}

		memcpy(&item->node, node, sizeof(*node));
		item->node.name = NULL;
		list_add_tail(&cur_list->node->value.list, &item->list);
	} else {
		struct toml_table_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}
		memcpy(&item->node, node, sizeof(*node));
		item->node.name = name;
		list_add_tail(&cur_table->value.map, &item->map);
	}

	return true;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - ('#'|'='|'"'|whitespace))+					>{ts = p;};
	name_in_double_quotes = (print - '"')+						>{ts = p;};
	name_in_single_quotes = (print - "'")+						>{ts = p;};
	tablename =  (print - ('#'|']'|'['|'"'|whitespace))+		>{ts = p;};
	tablename_in_double_quotes =  (print - '"')+				>{ts = p;};
	tablename_in_single_quotes =  (print - "'")+				>{ts = p;};

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen + 1) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, cur_line);
			fbreak;
		}

		while (ts[namelen] == ' ' || ts[namelen] == '\t')
			namelen--;

		name = strndup(ts, namelen + 1);
	}

	action saw_bool {
		struct toml_node node;

		node.type = TOML_BOOLEAN;
		node.value.integer = number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_int {
		char*				te = p;
		struct toml_node	node;

		node.type = TOML_INT;
		node.value.integer = negative ? -number : number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_float {
		char*				te = p;
		struct toml_node	node;

		floating = strtod(ts, &te);

		node.type = TOML_FLOAT;
		node.value.floating.value = floating;
		node.value.floating.precision = precision;

		if (!node.value.floating.precision && !exponent) {
			asprintf(&parse_error, "bad float\n");
			fbreak;
		}

		exponent = false;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_string {
		int					len = strp - string + 1;
		struct toml_node	node;

		*strp = 0;

		node.type = TOML_STRING;
		node.value.string = malloc(len);
		if (!node.value.string) {
			malloc_error = 1;
			fbreak;
		}
		memcpy(node.value.string, string, len);

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_date {
		char*	te = p;
		struct	toml_node node;

		node.type = TOML_DATE;
		node.value.rfc3339_time.epoch = timegm(&tm);
		node.value.rfc3339_time.offset_sign_negative = time_offset_is_negative;
		node.value.rfc3339_time.offset = time_offset;
		node.value.rfc3339_time.offset_is_zulu = time_offset_is_zulu;
		if (secfrac_ptr)
			node.value.rfc3339_time.sec_frac = strtol(secfrac_ptr, &te, 10);
		else
			node.value.rfc3339_time.sec_frac = -1;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), cur_line);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_inline_table {
		char*					tablename = name;
		struct toml_table_item*	item;
		struct toml_node*		place = cur_table ? cur_table : toml_root;
		bool					found = false;

		inline_table = true;

		list_for_each(&place->value.map, item, map) {
			if (strcmp(item->node.name, tablename) != 0)
				continue;

			found = true;
			break;
		}

		if (found)
		{
			asprintf(&parse_error, "duplicate entry %s\n", tablename);
			fbreak;
		}

		item = malloc(sizeof(*item));
		if (!item) {
			malloc_error = 1;
			fbreak;
		}

		item->node.name = strdup(tablename);
		item->node.type = TOML_TABLE;
		list_head_init(&item->node.value.map);
		list_add_tail(&place->value.map, &item->map);

		cur_table = &item->node;

		free(tablename);
	}

	action saw_table {
		char *ancestor, *tofree = NULL, *tablename;
		int item_added = 0;
		int len = (int)(p-ts);

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, len);

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", len, ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s", len, ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		int		ret;
		char*	tableName;

		tableName = strndup(ts, (int)(p-ts-1));
		ret = SawTableArray(toml_root, tableName, &cur_table, &parse_error);
		free(tableName);
		if (ret)
			fbreak;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	action bad_escape {
		asprintf(&parse_error, "bad escape \\%c", *p);
		fbreak;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			'#'			>{in_text = 0; fcall comment;}				->start	|
			[\t ]+		>{in_text = 0; indent = 0;} @{indent++;}	->start	|
			[\n]		>{in_text = 0; cur_line++;}					->start	|
			[\0]		>{in_text = 0; fbreak;}						->start	|
			[^\t \n]	@{fhold;} %{in_text = 1;}					->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] ${cur_line++; fret;} ),

		# a table
		table: (
			tablename ']' @saw_table										->start	|
			'"' tablename_in_double_quotes '"' @saw_table ']'				->start	|
			"'" tablename_in_single_quotes "'" @saw_table ']'				->start	|
			'[' tablename ']' ']' @saw_table_array							->start	|
			'[' '"' tablename_in_double_quotes '"' ']' @saw_table_array ']'	->start	|
			'[' "'" tablename_in_single_quotes "'" ']' @saw_table_array ']'	->start
		),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]				-> basic_empty_or_multi_line	|
			[^"] ${fhold;}	-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]							-> basic_multi_line_start	|
			[^"] $saw_string ${fhold;}	-> start
		),

		basic_multi_line_start: (
			'\n' ${cur_line++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${cur_line++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${cur_line++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${cur_line++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${cur_line++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'"'	${*strp++='"';}			${fret;}	|
			'/'	${*strp++='/';}			${fret;}	|
			'\\'	${*strp++='\\';}	${fret;}	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0uU"/\\] $bad_escape
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			'_' ? digit ${number *= 10; number += fc-'0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}				->date				|
			[eE]											->exponent_part		|
			[.]	>{precision = 0;}							->fractional_part	|
			[\t ,\]\n\0] $saw_int ${fhold;}					->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9]	${precision++;}				->fractional_part	|
			[eE]								->exponent_part		|
			[^0-9eE]	$saw_float ${fhold;}	->start
		),

		exponent_part: (
			[\-\+0-9]	${exponent=true;}		->exponent_part	|
			[^\-\+0-9]	$saw_float ${fhold;}	->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: ( '' >{secfrac_ptr = NULL; time_offset = 0; time_offset_is_negative = 0; time_offset_is_zulu = 0;}
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);} -> fractional_second_or_offset
		),

		fractional_second_or_offset: (
			'.' digit* >{secfrac_ptr=p;} >{fhold;}	-> time_offset	|
			[^.] @{fhold;}							-> time_offset
		),

		time_offset: (
			('-' @{time_offset_is_negative=1;}|'+')
				digit{2} @{time_offset = atoi(fpc-1) * 60;}
				':'
				digit{2} @{time_offset += atoi(fpc-1);}
				@saw_date								-> start |
			'Z' >{time_offset_is_zulu = 1;} @saw_date	-> start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${cur_line++;}	-> literal_multi_line	|
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${cur_line++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string							-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}	-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'		@{number = 1;}									-> true				|
			'false'		@{number = 0;}									-> false			|
			'"'			${strp = string;}								-> basic_string		|
			[']			${strp = string;}								-> literal_string	|
			('-'|'+')	${negative = fc == '-'; number = 0; ts = p;}	-> number_or_date	|
			digit		${negative = false; ts = p; number = fc-'0';}	-> number_or_date
		),

		# A list of values
		list: (
			'#'		>{fcall comment;}	->list	|
			'\n'	${cur_line++;}		->list	|
			[\t ]						->list	|
			','							->list	|
			']'	$end_list				->start	|
			[^#\t, \n\]] ${fhold;}		->val
		),

		inline_table: (
			','								-> inline_table |
			[\t ]							-> inline_table |
			'}'	@{inline_table = false;}	-> start		|
			[^\t ,}] @{fhold;}				-> key
		),

		# A val can be either a list or a singular value
		val: (
			'#'	>{fcall comment;}	->val			|
			'\n' ${ cur_line++; }	->val			|
			[\t ]					->val			|
			'['	$start_list			->list			|
			']'	$end_list			->start			|
			'{'	$saw_inline_table	->inline_table	|
			[^#\t \n[\]{] ${fhold;}	->singular
		)

		# A regular key
		key: (
			name @{namelen = (int)(p-ts);} whitespace '=' $saw_key								->val	|
			'"' name_in_double_quotes '"' @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val	|
			"'" name_in_single_quotes "'" @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'	>{fcall comment;}	->text	|
			'['						->table	|
			[\t ]					->text	|
			'\n' ${cur_line++;}		->start	|
			[^#[\t \n]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node* toml_root, char* buf, int buflen)
{
	int indent = 0, cs, cur_line = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int precision;
	int namelen;
	int64_t number;
	bool negative;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];
	int in_text = 0;
	int time_offset = 0;
	char* secfrac_ptr;
	bool time_offset_is_negative = 0;
	bool time_offset_is_zulu = 0;
	bool inline_table = false;
	bool exponent = false;

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen + 1;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", cur_line);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, cur_line, p);
		free(parse_error);
		return 1;
	}

	if (in_text) {
		fprintf(stderr, "not in start, line %d\n", cur_line);
		return 1;
	}

	/* check we have consumed the entire buffer */
	if (p != pe) {
		fprintf(stderr, "entire buffer unconsumed, line %d\n", cur_line);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", cur_line, p);
		return 1;
	}

	return 0;
}


Fix typos introduced in CL9207

Fix typos introduced in CL9207

~$0x1f => $~0x1f
enum with kNoRestrictedReg is not removed from ValidateChunkAMD64

BUG=http://code.google.com/p/nativeclient/issues/detail?id=2597
TEST=trybots
Review URL: https://chromiumcodereview.appspot.com/10815055

git-svn-id: 721b910a23eff8a86f00c8fd261a7587cddf18f8@9227 fcba33aa-ac0c-11dd-b9e7-8d5594d729c2

/*
 * Copyright (c) 2012 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "native_client/src/trusted/validator_ragel/unreviewed/validator_internal.h"

#include "native_client/src/trusted/validator_ragel/gen/validator_x86_64_instruction_consts.c"

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action check_access {
    check_access(begin - data, base, index, restricted_register, valid_targets,
                 &errors_detected);
  }

  action rel8_operand {
    rel8_operand(p + 1, data, jump_dests, size, &errors_detected);
  }
  action rel16_operand {
    #error rel16_operand should never be used in nacl
  }
  action rel32_operand {
    rel32_operand(p + 1, data, jump_dests, size, &errors_detected);
  }

  # Do nothing when IMM operand is detected for now.  Will be used later for
  # dynamic code modification support.
  action imm2_operand { }
  action imm8_operand { }
  action imm16_operand { }
  action imm32_operand { }
  action imm64_operand { }
  action imm8_second_operand { }
  action imm16_second_operand { }
  action imm32_second_operand { }
  action imm64_second_operand { }

  action process_0_operands {
    process_0_operands(&restricted_register, &errors_detected);
  }
  action process_1_operands {
    process_1_operands(&restricted_register, &errors_detected, rex_prefix,
                       operand_states, begin, &sandboxed_rsi_restricted_rdi);
  }
  action process_2_operands {
    process_2_operands(&restricted_register, &errors_detected, rex_prefix,
                       operand_states, begin, &sandboxed_rsi_restricted_rdi);
  }

  include decode_x86_64 "validator_x86_64_instruction.rl";

  data16condrep = (data16 | condrep data16 | data16 condrep);
  data16rep = (data16 | rep data16 | data16 rep);

  # Special %rbp modifications without required sandboxing
  rbp_modifications =
    (0x48 0x89 0xe5)                       | # mov %rsp,%rbp
    (0x48 0x81 0xe5 any{3} (0x80 .. 0xff)) | # and $XXX,%rbp
    (0x48 0x83 0xe5 (0x80 .. 0xff))          # and $XXX,%rbp
    @process_0_operands;

  # Special instructions used for %rbp sandboxing
  rbp_sandboxing =
    (0x4c 0x01 0xfd            | # add %r15,%rbp
     0x49 0x8d 0x2c 0x2f       | # lea (%r15,%rbp,1),%rbp
     0x4a 0x8d 0x6c 0x3d 0x00)   # lea 0x0(%rbp,%r15,1),%rbp
    @{ if (restricted_register != REG_RBP) {
         errors_detected |= RESTRICTED_RBP_UNPROCESSED;
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    };

  # Special %rbp modifications without required sandboxing
  rsp_modifications =
    (0x48 0x89 0xec)                       | # mov %rbp,%rsp
    (0x48 0x81 0xe4 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe4 (0x80 .. 0xff))          # and $XXX,%rsp
    @process_0_operands;

  # Special instructions used for %rbp sandboxing
  rsp_sandboxing =
    (0x4c 0x01 0xfc       | # add %r15,%rsp
     0x4a 0x8d 0x24 0x3c)   # lea (%rsp,%r15,1),%rsp
    @{ if (restricted_register != REG_RSP) {
         errors_detected |= RESTRICTED_RSP_UNPROCESSED;
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    };

  # naclcall - old version (is not used by contemporary toolchain)
  # Note: first "and $~0x1f, %eXX" is normal instruction and as such will detect
  # case where %rbp/%rsp is illegally modified.
  old_naclcall_or_nacljmp =
    (((0x83 0xe0 0xe0 0x49 0x8d any 0x07 any{2}) | # and $~0x1f, %eax
      (0x83 0xe1 0xe0 0x49 0x8d any 0x0f any{2}) | #   lea (%r15,%rax),%rXX
      (0x83 0xe2 0xe0 0x49 0x8d any 0x17 any{2}) | # ...
      (0x83 0xe3 0xe0 0x49 0x8d any 0x1f any{2}) | # ...
      (0x83 0xe6 0xe0 0x49 0x8d any 0x37 any{2}) | # and $~0x1f, %edi
      (0x83 0xe7 0xe0 0x49 0x8d any 0x3f any{2}) | #   lea (%r15,%rdi),%rXX
      (0x41 0x83 0xe0 0xe0 0x49 0x8d any 0x07 any{2}) | # and $~0x1f, %r8d
      (0x41 0x83 0xe1 0xe0 0x49 0x8d any 0x0f any{2}) | #   lea (%r15,%rax),%rXX
      (0x41 0x83 0xe2 0xe0 0x49 0x8d any 0x17 any{2}) | #
      (0x41 0x83 0xe3 0xe0 0x49 0x8d any 0x1f any{2}) | # ...
      (0x41 0x83 0xe4 0xe0 0x49 0x8d any 0x27 any{2}) | #
      (0x41 0x83 0xe5 0xe0 0x49 0x8d any 0x2f any{2}) | # and $~0x1f, %r14d
      (0x41 0x83 0xe6 0xe0 0x49 0x8d any 0x37 any{2})) & #  lea (%r15,%rdi),%rXX
     ((any{3,4} 0x49 0x8d 0x04 any 0xff (0xd0|0xe0)) | # lea (%r15,%rXX),%rax
      (any{3,4} 0x49 0x8d 0x0c any 0xff (0xd1|0xe1)) | #   call/jmp %rax
      (any{3,4} 0x49 0x8d 0x14 any 0xff (0xd2|0xe2)) | # ...
      (any{3,4} 0x49 0x8d 0x1c any 0xff (0xd3|0xe3)) | # ...
      (any{3,4} 0x49 0x8d 0x34 any 0xff (0xd6|0xe6)) | # lea (%r15,%rXX),%rdi
      (any{3,4} 0x49 0x8d 0x3c any 0xff (0xd7|0xe7)))) #   call/jmp %rdi
    @{
       BitmapClearBit(valid_targets, (p - data) - 6);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (((0x83 0xe0 0xe0 0x4d 0x8d any 0x07 any{3})  | # and $~0x1f, %eax
      (0x83 0xe1 0xe0 0x4d 0x8d any 0x0f any{3})  | #   lea (%r15,%rax),%rXX
      (0x83 0xe2 0xe0 0x4d 0x8d any 0x17 any{3})  | # ...
      (0x83 0xe3 0xe0 0x4d 0x8d any 0x1f any{3})  | # ...
      (0x83 0xe6 0xe0 0x4d 0x8d any 0x37 any{3})  | # and $~0x1f, %edi
      (0x83 0xe7 0xe0 0x4d 0x8d any 0x3f any{3})  | #   lea (%r15,%rdi),%rXX
      (0x41 0x83 0xe0 0xe0 0x4d 0x8d any 0x07 any{3}) | # and $~0x1f, %r8d
      (0x41 0x83 0xe1 0xe0 0x4d 0x8d any 0x0f any{3}) | #   lea (%r15,%rax),%rXX
      (0x41 0x83 0xe2 0xe0 0x4d 0x8d any 0x17 any{3}) | #
      (0x41 0x83 0xe3 0xe0 0x4d 0x8d any 0x1f any{3}) | # ...
      (0x41 0x83 0xe4 0xe0 0x4d 0x8d any 0x27 any{3}) | #
      (0x41 0x83 0xe5 0xe0 0x4d 0x8d any 0x2f any{3}) | # and $~0x1f, %r14d
      (0x41 0x83 0xe6 0xe0 0x4d 0x8d any 0x37 any{3})) & #  lea (%r15,%rdi),%rXX
     ((any{3,4} 0x4d 0x8d 0x04 any 0x41 0xff (0xd0|0xe0)) | # lea (%r15,%rXX),
      (any{3,4} 0x4d 0x8d 0x0c any 0x41 0xff (0xd1|0xe1)) | #                %r8
      (any{3,4} 0x4d 0x8d 0x14 any 0x41 0xff (0xd2|0xe2)) | #   call/jmp %r8
      (any{3,4} 0x4d 0x8d 0x1c any 0x41 0xff (0xd3|0xe3)) | # ...
      (any{3,4} 0x4d 0x8d 0x24 any 0x41 0xff (0xd4|0xe4)) | # lea (%r15,%rXX),
      (any{3,4} 0x4d 0x8d 0x2c any 0x41 0xff (0xd5|0xe5)) | #               %r14
      (any{3,4} 0x4d 0x8d 0x34 any 0x41 0xff (0xd6|0xe6)))) #   call/jmp %r14
    @{
       BitmapClearBit(valid_targets, (p - data) - 7);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    };

  # naclcall or nacljmp - this the form used by contemporary toolchain
  # Note: first "and $~0x1f, %eXX" is normal instruction and as such will detect
  # case where %rbp/%rsp is illegally modified.
  naclcall_or_nacljmp =
    (0x83 0xe0 0xe0 0x4c 0x01 0xf8 0xff (0xd0|0xe0) | # naclcall/jmp %eax, %r15
     0x83 0xe1 0xe0 0x4c 0x01 0xf9 0xff (0xd1|0xe1) | # naclcall/jmp %ecx, %r15
     0x83 0xe2 0xe0 0x4c 0x01 0xfa 0xff (0xd2|0xe2) | # naclcall/jmp %edx, %r15
     0x83 0xe3 0xe0 0x4c 0x01 0xfb 0xff (0xd3|0xe3) | # naclcall/jmp %ebx, %r15
     0x83 0xe6 0xe0 0x4c 0x01 0xfe 0xff (0xd6|0xe6) | # naclcall/jmp %esi, %r15
     0x83 0xe7 0xe0 0x4c 0x01 0xff 0xff (0xd7|0xe7))  # naclcall/jmp %edi, %r15
    @{
       BitmapClearBit(valid_targets, (p - data) - 4);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (0x41 0x83 0xe0 0xe0 0x4d 0x01 0xf8 0x41 0xff (0xd0|0xe0) | # naclcall/jmp
     0x41 0x83 0xe1 0xe0 0x4d 0x01 0xf9 0x41 0xff (0xd1|0xe1) | #    %r8d, %r15
     0x41 0x83 0xe2 0xe0 0x4d 0x01 0xfa 0x41 0xff (0xd2|0xe2) | #
     0x41 0x83 0xe3 0xe0 0x4d 0x01 0xfb 0x41 0xff (0xd3|0xe3) | # ...
     0x41 0x83 0xe4 0xe0 0x4d 0x01 0xfc 0x41 0xff (0xd4|0xe4) | #
     0x41 0x83 0xe5 0xe0 0x4d 0x01 0xfd 0x41 0xff (0xd5|0xe5) | # naclcall/jmp
     0x41 0x83 0xe6 0xe0 0x4d 0x01 0xfe 0x41 0xff (0xd6|0xe6))  #   %r14d, %r15
    @{
       BitmapClearBit(valid_targets, (p - data) - 5);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    };

  # Special %rsi sandboxing for string instructions
  rsi_sandboxing =
    (0x49 0x8d 0x34 0x37) # lea (%r15,%rsi,1),%rsi
    @{ if (restricted_register == REG_RSI) {
         sandboxed_rsi = begin;
         restricted_register = kSandboxedRsi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    };

  # Special %rdi sandboxing for string instructions
  rdi_sandboxing =
    (0x49 0x8d 0x3c 0x3f) # lea (%r15,%rdi,1),%rdi
    @{ if (restricted_register == REG_RDI) {
         sandboxed_rdi = begin;
         restricted_register = kSandboxedRdi;
       } else if (restricted_register == kSandboxedRsiRestrictedRdi) {
         sandboxed_rdi = begin;
         restricted_register = kSandboxedRsiSandboxedRdi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    };

  # String instructions which use only %ds:(%rsi)
  string_instructions_rsi_no_rdi =
    (rep? 0xac                 | # lods   %ds:(%rsi),%al
     data16rep 0xad            | # lods   %ds:(%rsi),%ax
     rep? REXW_NONE? 0xad)       # lods   %ds:(%rsi),%eax/%rax
    @{ if (restricted_register != kSandboxedRsi) {
         errors_detected |= RSI_UNSANDBOXDED;
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  # String instructions which use only %ds:(%rdi)
  string_instructions_rdi_no_rsi =
    (condrep? 0xae             | # scas   %es:(%rdi),%al
     data16condrep 0xaf        | # scas   %es:(%rdi),%ax
     condrep? REXW_NONE? 0xaf  | # scas   %es:(%rdi),%eax/%rax

     rep? 0xaa                 | # stos   %al,%es:(%rdi)
     data16rep 0xab            | # stos   %ax,%es:(%rdi)
     rep? REXW_NONE? 0xab)       # stos   %eax/%rax,%es:(%rdi)
    @{ if (restricted_register != kSandboxedRdi &&
           restricted_register != kSandboxedRsiSandboxedRdi) {
         errors_detected |= RDI_UNSANDBOXDED;
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  # String instructions which use both %ds:(%rsi) and %ds:(%rdi)
  string_instructions_rsi_rdi =
    (condrep? 0xa6            | # cmpsb    %es:(%rdi),%ds:(%rsi)
     data16condrep 0xa7       | # cmpsw    %es:(%rdi),%ds:(%rsi)
     condrep? REXW_NONE? 0xa7 | # cmps[lq] %es:(%rdi),%ds:(%rsi)

     rep? 0xa4                | # movsb    %es:(%rdi),%ds:(%rsi)
     data16rep 0xa5           | # movsw    %es:(%rdi),%ds:(%rsi)
     rep? REXW_NONE? 0xa5)      # movs[lq] %es:(%rdi),%ds:(%rsi)
    @{ if (restricted_register != kSandboxedRsiSandboxedRdi) {
         errors_detected |= RDI_UNSANDBOXDED;
         if (restricted_register != kSandboxedRsi) {
           errors_detected |= RSI_UNSANDBOXDED;
         }
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rsi - data));
         BitmapClearBit(valid_targets, (sandboxed_rsi_restricted_rdi - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  special_instruction =
    rbp_modifications |
    rsp_modifications |
    rbp_sandboxing |
    rsp_sandboxing |
    old_naclcall_or_nacljmp |
    naclcall_or_nacljmp |
    rsi_sandboxing |
    rdi_sandboxing |
    string_instructions_rsi_no_rdi |
    string_instructions_rdi_no_rsi |
    string_instructions_rsi_rdi;

  # Remove special instructions which are only allowed in special cases.
  normal_instruction = one_instruction - special_instruction;

  main := ((normal_instruction | special_instruction) >{
        begin = p;
        errors_detected = 0;
        BitmapSetBit(valid_targets, p - data);
        SET_REX_PREFIX(FALSE);
        SET_VEX_PREFIX2(0xe0);
        SET_VEX_PREFIX3(0x00);
        operand_states = 0;
     }
     @{
       if (errors_detected) {
         process_error(begin, errors_detected, userdata);
         result = 1;
       }
       /* On successful match the instruction start must point to the next byte
        * to be able to report the new offset as the start of instruction
        * causing error.  */
       begin = p + 1;
     })*
    $err{
        process_error(begin, UNRECOGNIZED_INSTRUCTION, userdata);
        result = 1;
        goto error_detected;
    };

}%%

%% write data;

int ValidateChunkAMD64(const uint8_t *data, size_t size,
                       const NaClCPUFeaturesX86 *cpu_features,
                       process_validation_error_func process_error,
                       void *userdata) {
  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
  const uint8_t *begin = p;  /* Start of the instruction being processed.  */

  uint8_t rex_prefix = FALSE;
  uint8_t vex_prefix2 = 0xe0;
  uint8_t vex_prefix3 = 0x00;
  /* Keeps one byte of information per operand in the current instruction:
   *  2 bits for register kinds,
   *  5 bits for register numbers (16 regs plus RIZ). */
  uint32_t operand_states = 0;
  enum register_name base = NO_REG;
  enum register_name index = NO_REG;
  int result = 0;
  /* These are borders of the appropriate instructions.  Initialize them to make
   * compiler happy: they are never used uninitialized even without explicit
   * initialization but GCC is not sophysicated enough to prove that.  */
  const uint8_t *sandboxed_rsi = 0;
  const uint8_t *sandboxed_rsi_restricted_rdi = 0;
  const uint8_t *sandboxed_rdi = 0;

  size_t i;

  uint32_t errors_detected = 0;

  assert(size % kBundleSize == 0);

  while (p < data + size) {
    const uint8_t *pe = p + kBundleSize;
    const uint8_t *eof = pe;
    int cs;
    uint8_t restricted_register = kNoRestrictedReg;

    %% write init;
    %% write exec;

    if (restricted_register == REG_RBP) {
      process_error(begin, RESTRICTED_RBP_UNPROCESSED, userdata);
      result = 1;
    } else if (restricted_register == REG_RSP) {
      process_error(begin, RESTRICTED_RSP_UNPROCESSED, userdata);
      result = 1;
    }
  }

  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      process_error(data + i * 32, BAD_JUMP_TARGET, userdata);
      result = 1;
      break;
    }
  }

error_detected:
  return result;
}

/*
 * Copyright (c) 2012 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "native_client/src/trusted/validator_ragel/unreviewed/validator_internal.h"

#include "native_client/src/trusted/validator_ragel/gen/validator_x86_64_instruction_consts.c"

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action check_access {
    check_access(begin - data, base, index, restricted_register, valid_targets,
                 &errors_detected);
  }

  action rel8_operand {
    rel8_operand(p + 1, data, jump_dests, size, &errors_detected);
  }
  action rel16_operand {
    #error rel16_operand should never be used in nacl
  }
  action rel32_operand {
    rel32_operand(p + 1, data, jump_dests, size, &errors_detected);
  }

  # Do nothing when IMM operand is detected for now.  Will be used later for
  # dynamic code modification support.
  action imm2_operand { }
  action imm8_operand { }
  action imm16_operand { }
  action imm32_operand { }
  action imm64_operand { }
  action imm8_second_operand { }
  action imm16_second_operand { }
  action imm32_second_operand { }
  action imm64_second_operand { }

  action process_0_operands {
    process_0_operands(&restricted_register, &errors_detected);
  }
  action process_1_operands {
    process_1_operands(&restricted_register, &errors_detected, rex_prefix,
                       operand_states, begin, &sandboxed_rsi_restricted_rdi);
  }
  action process_2_operands {
    process_2_operands(&restricted_register, &errors_detected, rex_prefix,
                       operand_states, begin, &sandboxed_rsi_restricted_rdi);
  }

  include decode_x86_64 "validator_x86_64_instruction.rl";

  data16condrep = (data16 | condrep data16 | data16 condrep);
  data16rep = (data16 | rep data16 | data16 rep);

  # Special %rbp modifications without required sandboxing
  rbp_modifications =
    (0x48 0x89 0xe5)                       | # mov %rsp,%rbp
    (0x48 0x81 0xe5 any{3} (0x80 .. 0xff)) | # and $XXX,%rbp
    (0x48 0x83 0xe5 (0x80 .. 0xff))          # and $XXX,%rbp
    @process_0_operands;

  # Special instructions used for %rbp sandboxing
  rbp_sandboxing =
    (0x4c 0x01 0xfd            | # add %r15,%rbp
     0x49 0x8d 0x2c 0x2f       | # lea (%r15,%rbp,1),%rbp
     0x4a 0x8d 0x6c 0x3d 0x00)   # lea 0x0(%rbp,%r15,1),%rbp
    @{ if (restricted_register != REG_RBP) {
         errors_detected |= RESTRICTED_RBP_UNPROCESSED;
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    };

  # Special %rbp modifications without required sandboxing
  rsp_modifications =
    (0x48 0x89 0xec)                       | # mov %rbp,%rsp
    (0x48 0x81 0xe4 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe4 (0x80 .. 0xff))          # and $XXX,%rsp
    @process_0_operands;

  # Special instructions used for %rbp sandboxing
  rsp_sandboxing =
    (0x4c 0x01 0xfc       | # add %r15,%rsp
     0x4a 0x8d 0x24 0x3c)   # lea (%rsp,%r15,1),%rsp
    @{ if (restricted_register != REG_RSP) {
         errors_detected |= RESTRICTED_RSP_UNPROCESSED;
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    };

  # naclcall - old version (is not used by contemporary toolchain)
  # Note: first "and ~$0x1f, %eXX" is normal instruction and as such will detect
  # case where %rbp/%rsp is illegally modified.
  old_naclcall_or_nacljmp =
    (((0x83 0xe0 0xe0 0x49 0x8d any 0x07 any{2}) | # and ~$0x1f, %eax
      (0x83 0xe0 0xe1 0x49 0x8d any 0x0f any{2}) | #   lea (%r15,%rax),%rXX
      (0x83 0xe0 0xe2 0x49 0x8d any 0x17 any{2}) | # ...
      (0x83 0xe0 0xe3 0x49 0x8d any 0x1f any{2}) | # ...
      (0x83 0xe0 0xe6 0x49 0x8d any 0x37 any{2}) | # and ~$0x1f, %edi
      (0x83 0xe0 0xe7 0x49 0x8d any 0x3f any{2}) | #   lea (%r15,%rdi),%rXX
      (0x41 0x83 0xe0 0xe0 0x49 0x8d any 0x07 any{2}) | # and ~$0x1f, %r8d
      (0x41 0x83 0xe0 0xe1 0x49 0x8d any 0x0f any{2}) | #   lea (%r15,%rax),%rXX
      (0x41 0x83 0xe0 0xe2 0x49 0x8d any 0x17 any{2}) | #
      (0x41 0x83 0xe0 0xe3 0x49 0x8d any 0x1f any{2}) | # ...
      (0x41 0x83 0xe0 0xe4 0x49 0x8d any 0x27 any{2}) | #
      (0x41 0x83 0xe0 0xe5 0x49 0x8d any 0x2f any{2}) | # and ~$0x1f, %r14d
      (0x41 0x83 0xe0 0xe6 0x49 0x8d any 0x37 any{2})) & #  lea (%r15,%rdi),%rXX
     ((any{3,4} 0x49 0x8d 0x04 any 0xff (0xd0|0xe0)) | # lea (%r15,%rXX),%rax
      (any{3,4} 0x49 0x8d 0x0c any 0xff (0xd1|0xe1)) | #   call/jmp %rax
      (any{3,4} 0x49 0x8d 0x14 any 0xff (0xd2|0xe2)) | # ...
      (any{3,4} 0x49 0x8d 0x1c any 0xff (0xd3|0xe3)) | # ...
      (any{3,4} 0x49 0x8d 0x34 any 0xff (0xd6|0xe6)) | # lea (%r15,%rXX),%rdi
      (any{3,4} 0x49 0x8d 0x3c any 0xff (0xd7|0xe7)))) #   call/jmp %rdi
    @{
       BitmapClearBit(valid_targets, (p - data) - 6);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (((0x83 0xe0 0xe0 0x4d 0x8d any 0x07 any{3})  | # and ~$0x1f, %eax
      (0x83 0xe0 0xe1 0x4d 0x8d any 0x0f any{3})  | #   lea (%r15,%rax),%rXX
      (0x83 0xe0 0xe2 0x4d 0x8d any 0x17 any{3})  | # ...
      (0x83 0xe0 0xe3 0x4d 0x8d any 0x1f any{3})  | # ...
      (0x83 0xe0 0xe6 0x4d 0x8d any 0x37 any{3})  | # and ~$0x1f, %edi
      (0x83 0xe0 0xe7 0x4d 0x8d any 0x3f any{3})  | #   lea (%r15,%rdi),%rXX
      (0x41 0x83 0xe0 0xe0 0x4d 0x8d any 0x07 any{3}) | # and ~$0x1f, %r8d
      (0x41 0x83 0xe0 0xe1 0x4d 0x8d any 0x0f any{3}) | #   lea (%r15,%rax),%rXX
      (0x41 0x83 0xe0 0xe2 0x4d 0x8d any 0x17 any{3}) | #
      (0x41 0x83 0xe0 0xe3 0x4d 0x8d any 0x1f any{3}) | # ...
      (0x41 0x83 0xe0 0xe4 0x4d 0x8d any 0x27 any{3}) | #
      (0x41 0x83 0xe0 0xe5 0x4d 0x8d any 0x2f any{3}) | # and ~$0x1f, %r14d
      (0x41 0x83 0xe0 0xe6 0x4d 0x8d any 0x37 any{3})) & #  lea (%r15,%rdi),%rXX
     ((any{3,4} 0x4d 0x8d 0x04 any 0x41 0xff (0xd0|0xe0)) | # lea (%r15,%rXX),
      (any{3,4} 0x4d 0x8d 0x0c any 0x41 0xff (0xd1|0xe1)) | #                %r8
      (any{3,4} 0x4d 0x8d 0x14 any 0x41 0xff (0xd2|0xe2)) | #   call/jmp %r8
      (any{3,4} 0x4d 0x8d 0x1c any 0x41 0xff (0xd3|0xe3)) | # ...
      (any{3,4} 0x4d 0x8d 0x24 any 0x41 0xff (0xd4|0xe4)) | # lea (%r15,%rXX),
      (any{3,4} 0x4d 0x8d 0x2c any 0x41 0xff (0xd5|0xe5)) | #               %r14
      (any{3,4} 0x4d 0x8d 0x34 any 0x41 0xff (0xd6|0xe6)))) #   call/jmp %r14
    @{
       BitmapClearBit(valid_targets, (p - data) - 7);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    };

  # naclcall or nacljmp - this the form used by contemporary toolchain
  # Note: first "and ~$0x1f, %eXX" is normal instruction and as such will detect
  # case where %rbp/%rsp is illegally modified.
  naclcall_or_nacljmp =
    (0x83 0xe0 0xe0 0x4c 0x01 0xf8 0xff (0xd0|0xe0) | # naclcall/jmp %eax, %r15
     0x83 0xe1 0xe0 0x4c 0x01 0xf9 0xff (0xd1|0xe1) | # naclcall/jmp %ecx, %r15
     0x83 0xe2 0xe0 0x4c 0x01 0xfa 0xff (0xd2|0xe2) | # naclcall/jmp %edx, %r15
     0x83 0xe3 0xe0 0x4c 0x01 0xfb 0xff (0xd3|0xe3) | # naclcall/jmp %ebx, %r15
     0x83 0xe6 0xe0 0x4c 0x01 0xfe 0xff (0xd6|0xe6) | # naclcall/jmp %esi, %r15
     0x83 0xe7 0xe0 0x4c 0x01 0xff 0xff (0xd7|0xe7))  # naclcall/jmp %edi, %r15
    @{
       BitmapClearBit(valid_targets, (p - data) - 4);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (0x41 0x83 0xe0 0xe0 0x4d 0x01 0xf8 0x41 0xff (0xd0|0xe0) | # naclcall/jmp
     0x41 0x83 0xe1 0xe0 0x4d 0x01 0xf9 0x41 0xff (0xd1|0xe1) | #    %r8d, %r15
     0x41 0x83 0xe2 0xe0 0x4d 0x01 0xfa 0x41 0xff (0xd2|0xe2) | #
     0x41 0x83 0xe3 0xe0 0x4d 0x01 0xfb 0x41 0xff (0xd3|0xe3) | # ...
     0x41 0x83 0xe4 0xe0 0x4d 0x01 0xfc 0x41 0xff (0xd4|0xe4) | #
     0x41 0x83 0xe5 0xe0 0x4d 0x01 0xfd 0x41 0xff (0xd5|0xe5) | # naclcall/jmp
     0x41 0x83 0xe6 0xe0 0x4d 0x01 0xfe 0x41 0xff (0xd6|0xe6))  #   %r14d, %r15
    @{
       BitmapClearBit(valid_targets, (p - data) - 5);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    };

  # Special %rsi sandboxing for string instructions
  rsi_sandboxing =
    (0x49 0x8d 0x34 0x37) # lea (%r15,%rsi,1),%rsi
    @{ if (restricted_register == REG_RSI) {
         sandboxed_rsi = begin;
         restricted_register = kSandboxedRsi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    };

  # Special %rdi sandboxing for string instructions
  rdi_sandboxing =
    (0x49 0x8d 0x3c 0x3f) # lea (%r15,%rdi,1),%rdi
    @{ if (restricted_register == REG_RDI) {
         sandboxed_rdi = begin;
         restricted_register = kSandboxedRdi;
       } else if (restricted_register == kSandboxedRsiRestrictedRdi) {
         sandboxed_rdi = begin;
         restricted_register = kSandboxedRsiSandboxedRdi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    };

  # String instructions which use only %ds:(%rsi)
  string_instructions_rsi_no_rdi =
    (rep? 0xac                 | # lods   %ds:(%rsi),%al
     data16rep 0xad            | # lods   %ds:(%rsi),%ax
     rep? REXW_NONE? 0xad)       # lods   %ds:(%rsi),%eax/%rax
    @{ if (restricted_register != kSandboxedRsi) {
         errors_detected |= RSI_UNSANDBOXDED;
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  # String instructions which use only %ds:(%rdi)
  string_instructions_rdi_no_rsi =
    (condrep? 0xae             | # scas   %es:(%rdi),%al
     data16condrep 0xaf        | # scas   %es:(%rdi),%ax
     condrep? REXW_NONE? 0xaf  | # scas   %es:(%rdi),%eax/%rax

     rep? 0xaa                 | # stos   %al,%es:(%rdi)
     data16rep 0xab            | # stos   %ax,%es:(%rdi)
     rep? REXW_NONE? 0xab)       # stos   %eax/%rax,%es:(%rdi)
    @{ if (restricted_register != kSandboxedRdi &&
           restricted_register != kSandboxedRsiSandboxedRdi) {
         errors_detected |= RDI_UNSANDBOXDED;
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  # String instructions which use both %ds:(%rsi) and %ds:(%rdi)
  string_instructions_rsi_rdi =
    (condrep? 0xa6            | # cmpsb    %es:(%rdi),%ds:(%rsi)
     data16condrep 0xa7       | # cmpsw    %es:(%rdi),%ds:(%rsi)
     condrep? REXW_NONE? 0xa7 | # cmps[lq] %es:(%rdi),%ds:(%rsi)

     rep? 0xa4                | # movsb    %es:(%rdi),%ds:(%rsi)
     data16rep 0xa5           | # movsw    %es:(%rdi),%ds:(%rsi)
     rep? REXW_NONE? 0xa5)      # movs[lq] %es:(%rdi),%ds:(%rsi)
    @{ if (restricted_register != kSandboxedRsiSandboxedRdi) {
         errors_detected |= RDI_UNSANDBOXDED;
         if (restricted_register != kSandboxedRsi) {
           errors_detected |= RSI_UNSANDBOXDED;
         }
       } else {
         BitmapClearBit(valid_targets, (begin - data));
         BitmapClearBit(valid_targets, (sandboxed_rsi - data));
         BitmapClearBit(valid_targets, (sandboxed_rsi_restricted_rdi - data));
         BitmapClearBit(valid_targets, (sandboxed_rdi - data));
       }
       restricted_register = kNoRestrictedReg;
    };

  special_instruction =
    rbp_modifications |
    rsp_modifications |
    rbp_sandboxing |
    rsp_sandboxing |
    old_naclcall_or_nacljmp |
    naclcall_or_nacljmp |
    rsi_sandboxing |
    rdi_sandboxing |
    string_instructions_rsi_no_rdi |
    string_instructions_rdi_no_rsi |
    string_instructions_rsi_rdi;

  # Remove special instructions which are only allowed in special cases.
  normal_instruction = one_instruction - special_instruction;

  main := ((normal_instruction | special_instruction) >{
        begin = p;
        errors_detected = 0;
        BitmapSetBit(valid_targets, p - data);
        SET_REX_PREFIX(FALSE);
        SET_VEX_PREFIX2(0xe0);
        SET_VEX_PREFIX3(0x00);
        operand_states = 0;
     }
     @{
       if (errors_detected) {
         process_error(begin, errors_detected, userdata);
         result = 1;
       }
       /* On successful match the instruction start must point to the next byte
        * to be able to report the new offset as the start of instruction
        * causing error.  */
       begin = p + 1;
     })*
    $err{
        process_error(begin, UNRECOGNIZED_INSTRUCTION, userdata);
        result = 1;
        goto error_detected;
    };

}%%

%% write data;

int ValidateChunkAMD64(const uint8_t *data, size_t size,
                       const NaClCPUFeaturesX86 *cpu_features,
                       process_validation_error_func process_error,
                       void *userdata) {
  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
  const uint8_t *begin = p;  /* Start of the instruction being processed.  */

  uint8_t rex_prefix = FALSE;
  uint8_t vex_prefix2 = 0xe0;
  uint8_t vex_prefix3 = 0x00;
  /* Keeps one byte of information per operand in the current instruction:
   *  2 bits for register kinds,
   *  5 bits for register numbers (16 regs plus RIZ). */
  uint32_t operand_states = 0;
  enum register_name base = NO_REG;
  enum register_name index = NO_REG;
  int result = 0;
  /* These are borders of the appropriate instructions.  Initialize them to make
   * compiler happy: they are never used uninitialized even without explicit
   * initialization but GCC is not sophysicated enough to prove that.  */
  const uint8_t *sandboxed_rsi = 0;
  const uint8_t *sandboxed_rsi_restricted_rdi = 0;
  const uint8_t *sandboxed_rdi = 0;

  size_t i;

  uint32_t errors_detected = 0;

  assert(size % kBundleSize == 0);

  while (p < data + size) {
    const uint8_t *pe = p + kBundleSize;
    const uint8_t *eof = pe;
    int cs;
    enum {
      kNoRestrictedReg = 32,
      kSandboxedRsi,
      kSandboxedRdi,
      kSandboxedRsiRestrictedRdi,
      kSandboxedRsiSandboxedRdi
    }; uint8_t restricted_register = kNoRestrictedReg;

    %% write init;
    %% write exec;

    if (restricted_register == REG_RBP) {
      process_error(begin, RESTRICTED_RBP_UNPROCESSED, userdata);
      result = 1;
    } else if (restricted_register == REG_RSP) {
      process_error(begin, RESTRICTED_RSP_UNPROCESSED, userdata);
      result = 1;
    }
  }

  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      process_error(data + i * 32, BAD_JUMP_TARGET, userdata);
      result = 1;
      break;
    }
  }

error_detected:
  return result;
}


There is no need to keep the parser#mark around with an appended buffer

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
    raise BufferLimitExceeded if buffer.bytesize > buffer_size
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message { message = Stomp::Message.new(nil, nil) }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) $ buffer % write_body :> NULL;

  message = (command > mark_message) :> headers :> (dynamic_body @ finish_message);

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        buffer_size = state.buffer_size

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
        end

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size
        @error = nil
        @buffer = nil

        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    m = p
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
    raise BufferLimitExceeded if buffer.bytesize > buffer_size
  }
  action mark_key {
    mk = buffer # needs reset
    m = buffer = nil
  }
  action mark_message { message = Stomp::Message.new(nil, nil) }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    m = buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    m = mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    m = buffer = nil
  }

  action finish_headers {
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) $ buffer % write_body :> NULL;

  message = (command > mark_message) :> headers :> (dynamic_body @ finish_message);

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        m = state.mark # pointer to marked character (for data buffering)
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        buffer_size = state.buffer_size

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark = m
          state.mark_key = mk
          state.buffer = buffer
        end

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size
        @error = nil
        @buffer = nil

        @current_state = Stomp::Parser.start
        @message = nil
        @mark = nil
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [Integer] pointer to beginning of current buffer
      attr_accessor :mark

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end