Add null-terminated body parsing to JavaParser

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    mark_key = RubyString.newString(context.runtime, data, mark, p - mark);
    mark = -1;
  }

  action write_header {
    IRubyObject args[] = { mark_key, RubyString.newString(context.runtime, data, mark, p - mark) };
    mark_message.callMethod(context, "write_header", args);
    mark_key = null;
    mark = -1;
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    mark_message.callMethod(context, "write_body", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private class State {
    public int cs = JavaParser.start;
    public byte[] chunk;
    public int mark = -1;
    public RubyString mark_key;
    public IRubyObject mark_message;
    public int mark_message_size = -1;
    public int mark_content_length = -1;
  }

  private RaiseException exception;
  private long maxMessageSize;
  private State state;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
    state = new State();
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();

    int p = 0;
    int pe = data.length;

    int cs = state.cs;
    int mark = state.mark;
    RubyString mark_key = state.mark_key;
    IRubyObject mark_message = state.mark_message;
    int mark_message_size = state.mark_message_size;
    int mark_content_length = state.mark_content_length;

    %% write exec;

    state.cs = cs;
    state.mark = mark;
    state.mark_key = mark_key;
    state.mark_message = mark_message;
    state.mark_message_size = mark_message_size;
    state.mark_content_length = mark_content_length;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    mark_key = RubyString.newString(context.runtime, data, mark, p - mark);
    mark = -1;
  }

  action write_header {
    IRubyObject args[] = { mark_key, RubyString.newString(context.runtime, data, mark, p - mark) };
    mark_message.callMethod(context, "write_header", args);
    mark_key = null;
    mark = -1;
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    /*rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());*/
    /*mark = NULL;*/
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private class State {
    public int cs = JavaParser.start;
    public byte[] chunk;
    public int mark = -1;
    public RubyString mark_key;
    public IRubyObject mark_message;
    public int mark_message_size = -1;
    public int mark_content_length = -1;
  }

  private RaiseException exception;
  private long maxMessageSize;
  private State state;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
    state = new State();
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();

    int p = 0;
    int pe = data.length;

    int cs = state.cs;
    int mark = state.mark;
    RubyString mark_key = state.mark_key;
    IRubyObject mark_message = state.mark_message;
    int mark_message_size = state.mark_message_size;
    int mark_content_length = state.mark_content_length;

    %% write exec;

    state.cs = cs;
    state.mark = mark;
    state.mark_key = mark_key;
    state.mark_message = mark_message;
    state.mark_message_size = mark_message_size;
    state.mark_content_length = mark_content_length;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}


pass burntsushi test 'string-bad-escape'

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_TABLE);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	CASE_ENUM_TO_STR(TOML_TABLE_ARRAY);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'|'['))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, curline);
			fbreak;
		}
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string + 1;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_table {
		char *ancestor, *tofree, *tablename;
		int item_added = 0;

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", (int)(p-ts), ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		char *ancestor, *tofree, *tablename;

		struct toml_node *place = toml_root;
		struct toml_list_item *new_table_entry = NULL;

		tofree = tablename = strndup(ts, (int)(p-ts-1));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */

			/*
			 * Create a table array node and insert it into the heirarchy
			 */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE_ARRAY;
			list_head_init(&item->node.value.list);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		if (place->type != TOML_TABLE_ARRAY) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		/*
		 * Create a table which becomes the last element in the list
		 * of maps (table array is a list of maps)
		 */
		new_table_entry = malloc(sizeof(*new_table_entry));
		if (!new_table_entry) {
			malloc_error = 1;
			fbreak;
		}

		new_table_entry->node.type = TOML_TABLE;
		new_table_entry->node.name = NULL;
		list_head_init(&new_table_entry->node.value.map);
		list_add_tail(&place->value.list, &new_table_entry->list);

		cur_table = &new_table_entry->node;
	}

	action saw_comment {
		if (!list_empty(&list_stack))
			fnext list;

		curline++;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	action bad_escape {
		asprintf(&parse_error, "bad escape \\%c", *p);
		fbreak;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @saw_comment ->start ),

		# a table
		table: (
			name ']' @saw_table					->start	|
			'[' name ']' ']' @saw_table_array	->start
		),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any 	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]					-> basic_empty_or_multi_line	|
			[^"] ${fhold;}		-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]					-> basic_multi_line_start	|
			[^"] $saw_string	-> start
		),

		basic_multi_line_start: (
			'\n' ${curline++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${curline++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${curline++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${curline++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${curline++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'"'	${*strp++='"';}			${fret;}	|
			'x'							-> hex_byte	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0xuU"] $bad_escape
		),

		hex_byte: (
			xdigit{2} >{hex[0]=*p;}
				@{hex[1]=*p; *strp++=strtoul(hex, NULL, 16); fret;}
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${curline++;}	-> literal_multi_line |
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${curline++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string								-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}		-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true				|
			'false'	@{number = 0;}				-> false			|
			'"'		${strp = string;}			-> basic_string	|
			[']		${strp = string;}			-> literal_string	|
			('-'|'+') ${fhold;number = 0;}		-> sign				|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->list		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->table		|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	char hex[3] = { 0 };
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, curline, p);
		free(parse_error);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_TABLE);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	CASE_ENUM_TO_STR(TOML_TABLE_ARRAY);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'|'['))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, curline);
			fbreak;
		}
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string + 1;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_table {
		char *ancestor, *tofree, *tablename;
		int item_added = 0;

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", (int)(p-ts), ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		char *ancestor, *tofree, *tablename;

		struct toml_node *place = toml_root;
		struct toml_list_item *new_table_entry = NULL;

		tofree = tablename = strndup(ts, (int)(p-ts-1));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */

			/*
			 * Create a table array node and insert it into the heirarchy
			 */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE_ARRAY;
			list_head_init(&item->node.value.list);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		if (place->type != TOML_TABLE_ARRAY) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		/*
		 * Create a table which becomes the last element in the list
		 * of maps (table array is a list of maps)
		 */
		new_table_entry = malloc(sizeof(*new_table_entry));
		if (!new_table_entry) {
			malloc_error = 1;
			fbreak;
		}

		new_table_entry->node.type = TOML_TABLE;
		new_table_entry->node.name = NULL;
		list_head_init(&new_table_entry->node.value.map);
		list_add_tail(&place->value.list, &new_table_entry->list);

		cur_table = &new_table_entry->node;
	}

	action saw_comment {
		if (!list_empty(&list_stack))
			fnext list;

		curline++;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @saw_comment ->start ),

		# a table
		table: (
			name ']' @saw_table					->start	|
			'[' name ']' ']' @saw_table_array	->start
			),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any 	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]					-> basic_empty_or_multi_line	|
			[^"] ${fhold;}		-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]					-> basic_multi_line_start	|
			[^"] $saw_string	-> start
		),

		basic_multi_line_start: (
			'\n' ${curline++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${curline++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${curline++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${curline++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${curline++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'x'							-> hex_byte	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0xuU] ${*strp++=fc;}	${fret;}
		),

		hex_byte: (
			xdigit{2} >{hex[0]=*p;}
				@{hex[1]=*p; *strp++=strtoul(hex, NULL, 16); fret;}
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${curline++;}	-> literal_multi_line |
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${curline++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string								-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}		-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true				|
			'false'	@{number = 0;}				-> false			|
			'"'		${strp = string;}			-> basic_string	|
			[']		${strp = string;}			-> literal_string	|
			('-'|'+') ${fhold;number = 0;}		-> sign				|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->list		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->table		|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	char hex[3] = { 0 };
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, curline, p);
		free(parse_error);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}


take advantage of the fancy movable *pe and simplify the state graph

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action parse_dict { }
  action parse_arr {
    VALUE arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := tnets_size colon (
    tnets_dict |
    tnets_arr  |
    tnets_num  |
    tnets_str  |
    tnets_bool |
    tnets_null
  ) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

printf("%s\n", RSTRING_PTR(rb_funcall(parser->result, rb_intern("inspect"), 0)));
  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action parse_dict { }
  action parse_arr {
    VALUE arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
  }

  # parses the size spec at the start of the payload
  tnets_size = digit{,10} $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := tnets_size colon (
    tnets_dict |
    tnets_arr  |
    tnets_num  |
    tnets_str  |
    tnets_bool |
    tnets_null
  ) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

printf("%s\n", RSTRING_PTR(rb_funcall(parser->result, rb_intern("inspect"), 0)));
  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}


Fix regression of 544c854810bfdefe55d4e0aadaef3a8c3c168d01

%%{
  machine message;

  getkey data.getbyte(p); # code for retrieving current char

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action mark_key { mk = data.byteslice(m, p - m) }

  action write_command { message.write_command(data.byteslice(m, p - m)) }
  action write_header { message.write_header(mk, data.byteslice(m, p - m)) }
  action write_body { message.write_body(data.byteslice(m, p - m)) }

  action init_message {
    message = Stomp::Message.new
  }
  action finish_headers {}
  action finish_message {
    return message
  }

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark % mark_key;
  header_value = HEADER_OCTET* > mark;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) % write_body :> NULL;

  message := (command > init_message) :> headers :> (dynamic_body @ finish_message);
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = state.cursor # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        m = state.mark # pointer to marked character (for data buffering)
        mk = state.mark_key # key for header currently being read

        %% write exec;

        state.cursor = p
        state.message = message
        state.current_state = cs
        state.mark = m
        state.mark_key = mk

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size

        @cursor = 0
        @current_state = Stomp::Parser.start
        @message = nil
        @mark = 0
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [Integer] index of parsing cursor in data
      attr_accessor :cursor

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark_key

      # @return [Integer] index of parsing cursor in data
      # @return [Integer] index of parsing cursor in data
      def parse(data)
        Parser.parse(data, self)
      end
    end
  end
end

%%{
  machine message;

  getkey data.getbyte(p); # code for retrieving current char

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action mark_key { mk = data.byteslice(m, p) }

  action write_command { message.write_command(data.byteslice(m, p)) }
  action write_header { message.write_header(mk, data.byteslice(m, p)) }
  action write_body { message.write_body(data.byteslice(m, p)) }

  action init_message {
    message = Stomp::Message.new
  }
  action finish_headers {}
  action finish_message {
    return message
  }

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark % mark_key;
  header_value = HEADER_OCTET* > mark;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) % write_body :> NULL;

  message := (command > init_message) :> headers :> (dynamic_body @ finish_message);
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.length # end of chunk
        eof = :ignored # end of input

        p = state.cursor # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        m = state.mark # pointer to marked character (for data buffering)
        mk = state.mark_key # key for header currently being read

        %% write exec;

        state.cursor = p
        state.message = message
        state.current_state = cs
        state.mark = m
        state.mark_key = mk

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size

        @cursor = 0
        @current_state = Stomp::Parser.start
        @message = nil
        @mark = 0
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [Integer] index of parsing cursor in data
      attr_accessor :cursor

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark_key

      # @return [Integer] index of parsing cursor in data
      # @return [Integer] index of parsing cursor in data
      def parse(data)
        Parser.parse(data, self)
      end
    end
  end
end


remove comma and reduce generated code size

package httpstring

import (
	"strings"
	"time"
)

%% machine cache_control_parser;
%% write data;

// NewCacheControl is a struct type used for storing parsed tokens
func NewCacheControl(data []byte) *CacheControl {

	cs, p, pe := 0, 0, len(data)
	eof := pe

	str_beg := 0
	str_end := 0
	num, neg := 0, 1
	val := false

	cc := &CacheControl{
		strings: make(map[string]string),
		durations: make(map[string]time.Duration),
	}

	setbool := func(s string) {cc.setflag(s) }

	setstrings := func(k string) {
		setbool(k)
		cc.strings[strings.ToLower(k)] = string(data[str_beg:str_end])
	}
	setduration := func(k string) {
		setbool(k)
		v := time.Duration(int64(neg * num))
		cc.durations[k] = time.Second * v
		num = 0
		neg = 1
		val = false
	}

%%{
	action private          { setstrings("private") }
	action no_cache         { setstrings("no-cache") }

	action no_store         { setbool("no-store"); }
	action public           { setbool("public"); }

	action no_transform     { setbool("no-transform") }
	action only_if_cached   { setbool("only-if-cached") }
	action must_revalidate  { setbool("must-revalidate") }
	action proxy_revalidate { setbool("proxy-revalidate") }

	action max_age          { setduration("max-age") }
	action s_maxage         { setduration("s-maxage") }
	action min_fresh        { setduration("min-fresh") }

	action max_stale {
		setbool("max-stale");
		if val {
			setduration("max-stale")
		}
	}

	action str_init { str_beg = p }
	action str_end  { str_end = p }
	action sec_neg  { neg = -1 }
	action sec_incr { num = num * 10 + (int(fc)-'0') }

	tchar = [!#-'*-+\-.0-:@A-Z^-`a-z|~];
	value_token = tchar+ >str_init %str_end ;
	value_string = '"' ( [^"]* >str_init %str_end ) '"' ;
	value = '=' ( value_token | value_string ) ;
	integer = '0' | [1-9] digit* ;
	seconds = '=' ( ( "-" @sec_neg )? integer >{ val = true; } @sec_incr ) ;
	extension = ( tchar+ ) ( value )? ;

	def = "public"i %public
		| "private"i value? %private
		| "no-cache"i value? %no_cache
		| "no-store"i %no_store
		| "max-age"i seconds %max_age
		| "s-maxage"i seconds %s_maxage
		| "max-stale"i ( seconds )? %max_stale
		| "min-fresh"i seconds %min_fresh
		| "no-transform"i %no_transform
		| "only-if-cached"i %only_if_cached
		| "must-revalidate"i %must_revalidate
		| "proxy-revalidate"i %proxy_revalidate
		;

	ext = extension - def ;
	dir = ( def | ext );
	main := space* dir ( space* "," space* dir )* ;
	write init;
	write exec;
}%%

	return cc

}

package httpstring

import (
	"strings"
	"time"
)

%% machine cache_control_parser;
%% write data;

// NewCacheControl is a struct type used for storing parsed tokens
func NewCacheControl(data []byte) *CacheControl {

	cs, p, pe := 0, 0, len(data)
	eof := pe

	str_beg := 0
	str_end := 0
	num, neg := 0, 1
	val := false

	cc := &CacheControl{
		strings: make(map[string]string),
		durations: make(map[string]time.Duration),
	}

	setbool := func(s string) {cc.setflag(s) }

	setstrings := func(k string) {
		setbool(k)
		cc.strings[strings.ToLower(k)] = string(data[str_beg:str_end])
	}
	setduration := func(k string) {
		setbool(k)
		v := time.Duration(int64(neg * num))
		cc.durations[k] = time.Second * v
		num = 0
		neg = 1
		val = false
	}

%%{
	action private          { setstrings("private") }
	action no_cache         { setstrings("no-cache") }

	action no_store         { setbool("no-store"); }
	action public           { setbool("public"); }

	action no_transform     { setbool("no-transform") }
	action only_if_cached   { setbool("only-if-cached") }
	action must_revalidate  { setbool("must-revalidate") }
	action proxy_revalidate { setbool("proxy-revalidate") }

	action max_age          { setduration("max-age") }
	action s_maxage         { setduration("s-maxage") }
	action min_fresh        { setduration("min-fresh") }

	action max_stale {
		setbool("max-stale");
		if val {
			setduration("max-stale")
		}
	}

	action str_init { str_beg = p }
	action str_end  { str_end = p }
	action sec_neg  { neg = -1 }
	action sec_incr { num = num * 10 + (int(fc)-'0') }

	tchar = [!#-'*-.0-:@A-Z^-`a-z|~];
	value_token = tchar+ >str_init %str_end ;
	value_string = '"' ( [^"]* >str_init %str_end ) '"' ;
	value = '=' ( value_token | value_string ) ;
	integer = '0' | [1-9] digit* ;
	seconds = '=' ( ( "-" @sec_neg )? integer >{ val = true; } @sec_incr ) ;
	extension = ( tchar+ ) ( value )? ;

	def = "public"i %public
		| "private"i value? %private
		| "no-cache"i value? %no_cache
		| "no-store"i %no_store
		| "max-age"i seconds %max_age
		| "s-maxage"i seconds %s_maxage
		| "max-stale"i ( seconds )? %max_stale
		| "min-fresh"i seconds %min_fresh
		| "no-transform"i %no_transform
		| "only-if-cached"i %only_if_cached
		| "must-revalidate"i %must_revalidate
		| "proxy-revalidate"i %proxy_revalidate
		;

	ext = extension - def ;
	dir = ( def | ext );
	main := space* dir ( space* "," space* dir )* ;
	write init;
	write exec;
}%%

	return cc

}


do not print nullbyte in error message

do not print nullbyte in error message

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <iostream>
#include <iterator>
#include <algorithm>
#include <string>
#include <sstream>
#include <cassert>
#include <stdexcept>

#include "hext/token.h"

#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.tok_begin = p;       \
  tok.tok_end = nullptr;   \
  tokens.push_back(tok);

#define LX_TK_STOP                          \
  assert(tokens.size() > 0);                \
  tokens.at(tokens.size() - 1).tok_end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
}

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error()
  {
    typedef
      std::iterator_traits<std::string::const_iterator>::difference_type
      iter_diff_type;

    std::string all_to_error(this->p_begin, this->p);
    iter_diff_type line_nr =
      std::count(all_to_error.begin(), all_to_error.end(), '\n');

    size_t char_nr = 0;
    size_t line_offset = 0;
    if( line_nr )
    {
      line_offset = all_to_error.find_last_of('\n');
      if( line_offset == std::string::npos )
        line_offset = 1;
      // move past \n
      line_offset += 1;
    }
    char_nr = std::distance(this->p_begin, this->p) - line_offset;

    std::string error_token;
    if( *p == '\n' )
      error_token = "[newline]";
    else if( *p == ' ' )
      error_token = "[space]";
    else if( *p == '\0' )
      error_token = "[nullbyte]";
    else
      error_token = *p;

    std::stringstream error_msg;
    error_msg << "Error at line "
              << line_nr + 1
              << ", char "
              << char_nr + 1
              << ", unexpected character '"
              << error_token
              << "'";
    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <iostream>
#include <iterator>
#include <algorithm>
#include <string>
#include <sstream>
#include <cassert>
#include <stdexcept>

#include "hext/token.h"

#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.tok_begin = p;       \
  tok.tok_end = nullptr;   \
  tokens.push_back(tok);

#define LX_TK_STOP                          \
  assert(tokens.size() > 0);                \
  tokens.at(tokens.size() - 1).tok_end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
}

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error()
  {
    typedef
      std::iterator_traits<std::string::const_iterator>::difference_type
      iter_diff_type;

    std::string all_to_error(this->p_begin, this->p);
    iter_diff_type line_nr =
      std::count(all_to_error.begin(), all_to_error.end(), '\n');

    size_t char_nr = 0;
    size_t line_offset = 0;
    if( line_nr )
    {
      line_offset = all_to_error.find_last_of('\n');
      if( line_offset == std::string::npos )
        line_offset = 1;
      // move past \n
      line_offset += 1;
    }
    char_nr = std::distance(this->p_begin, this->p) - line_offset;

    std::string error_token;
    if( *p == '\n' )
      error_token = "[newline]";
    else if( *p == ' ' )
      error_token = "[space]";
    else
      error_token = *p;

    std::stringstream error_msg;
    error_msg << "Error at line "
              << line_nr + 1
              << ", char "
              << char_nr + 1
              << ", unexpected character '"
              << error_token
              << "'";
    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


impl a_list definition

impl a_list definition

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';

  action start_list {
    $stderr.puts "LIST start_list"
    @list_opened = true
    @incomplete[:list] = list()
  }

  action stop_list {
    $stderr.puts "LIST stop_list"
    @list_closed = true
  }

  action add_string {
    $stderr.puts "LIST add_string"
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    @incomplete[:list] <<= item
  }

  action add_ident {
    $stderr.puts "LIST add_ident"
    ident = @buffers.delete(:ident)
    item = list_item(ident, complete: ident.complete)
    @incomplete[:list] <<= item
  }

  action list_end {
    $stderr.puts "LIST list_end"
    if @list_opened && @list_closed
      list = @incomplete.delete(:list)
      list.complete = true
    elsif !@list_closed
      list = @incomplete.delete(:list)
      list.complete = false
    end
    @buffers[:list] = list
  }

  action list_node_eof {
    $stderr.puts "LIST list_node_eof"
    list = @incomplete.delete(:list)
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    list <<= item
    list.complete = false
    yield list
  }

  action yield_list {
    $stderr.puts "LIST yield_list"
    yield @buffers[:list]
  }

  START_LIST = '{' SP* >start_list;
  END_LIST = '}' %stop_list;

  string_item =
    a_string
    %add_string
    ;

  ident_item =
    an_ident
    %add_ident
    ;

  item =
    string_item |
    ident_item
    ;

  list_item_0 =
    item
    ;

  list_item_n =
    COMMA_DELIM
    item
    ;

  items =
    list_item_0
    list_item_n*
    SP*
    ;

  a_list =
    START_LIST
    items
    END_LIST
    %list_end
    ;

  list_node :=
    START_LIST
    items?
    @eof(list_node_eof)
    END_LIST?
    @eof(list_node_eof)
    NL?
    %list_end
    %yield_list
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers      = {}
            @incomplete   = {}
            @list_opened  = false
            @list_closed  = false
            data          = @content.unpack('C*')
            p             = 0
            pe            = data.length
            eof           = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';

  action start_list {
    $stderr.puts "LIST start_list"
    @list_opened = true
    @incomplete[:list] = list()
  }

  action stop_list {
    $stderr.puts "LIST stop_list"
    @list_closed = true
  }

  action add_string {
    $stderr.puts "LIST add_string"
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    @incomplete[:list] <<= item
  }

  action add_ident {
    $stderr.puts "LIST add_ident"
    ident = @buffers.delete(:ident)
    item = list_item(ident, complete: ident.complete)
    @incomplete[:list] <<= item
  }

  action list_end {
    $stderr.puts "LIST list_end"
    if @list_opened && @list_closed
      list = @incomplete.delete(:list)
      list.complete = true
    elsif !@list_closed
      list = @incomplete.delete(:list)
      list.complete = false
    end
    @buffers[:list] = list
  }

  action list_node_eof {
    $stderr.puts "LIST list_node_eof"
    list = @incomplete.delete(:list)
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    list <<= item
    list.complete = false
    yield list
  }

  action yield_list {
    $stderr.puts "LIST yield_list"
    yield @buffers[:list]
  }

  START_LIST = '{' SP* >start_list;
  END_LIST = '}' %stop_list;

  string_item =
    a_string
    %add_string
    ;

  ident_item =
    an_ident
    %add_ident
    ;

  item =
    string_item |
    ident_item
    ;

  list_item_0 =
    item
    ;

  list_item_n =
    COMMA_DELIM
    item
    ;

  items =
    list_item_0
    list_item_n*
    SP*
    ;

  a_list = START_LIST;

  list_node :=
    START_LIST
    items?
    @eof(list_node_eof)
    END_LIST?
    @eof(list_node_eof)
    NL?
    %list_end
    %yield_list
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers      = {}
            @incomplete   = {}
            @list_opened  = false
            @list_closed  = false
            data          = @content.unpack('C*')
            p             = 0
            pe            = data.length
            eof           = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Enforce that qstruct names begin with uppercase

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

ssize_t calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0);

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alnum | '_';
    uc_alpha_u = [A-Z] | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = uc_alpha_u alnum_u* ('::' uc_alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    ws*
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                    ws*
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec?
             $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); } ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  curr_def = NULL;

  while (def) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }

  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    packing_result = calculate_qstruct_packing(curr_def);

    if (packing_result < 0)
      PARSE_ERROR("memory error in packing");

    curr_def->body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
  }

  bail:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

ssize_t calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0);

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    ws*
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                    ws*
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec?
             $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); } ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  curr_def = NULL;

  while (def) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }

  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    packing_result = calculate_qstruct_packing(curr_def);

    if (packing_result < 0)
      PARSE_ERROR("memory error in packing");

    curr_def->body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
  }

  bail:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}