Pass burntsushi test 'empty-implicit-table'

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_TABLE);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	CASE_ENUM_TO_STR(TOML_TABLE_ARRAY);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'|'['))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, curline);
			fbreak;
		}
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string + 1;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_table {
		char *ancestor, *tofree, *tablename;
		int item_added = 0;

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", (int)(p-ts), ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		char *ancestor, *tofree, *tablename;

		struct toml_node *place = toml_root;
		struct toml_list_item *new_table_entry = NULL;

		tofree = tablename = strndup(ts, (int)(p-ts-1));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */

			/*
			 * Create a table array node and insert it into the heirarchy
			 */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE_ARRAY;
			list_head_init(&item->node.value.list);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		if (place->type != TOML_TABLE_ARRAY) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		/*
		 * Create a table which becomes the last element in the list
		 * of maps (table array is a list of maps)
		 */
		new_table_entry = malloc(sizeof(*new_table_entry));
		if (!new_table_entry) {
			malloc_error = 1;
			fbreak;
		}

		new_table_entry->node.type = TOML_TABLE;
		new_table_entry->node.name = NULL;
		list_head_init(&new_table_entry->node.value.map);
		list_add_tail(&place->value.list, &new_table_entry->list);

		cur_table = &new_table_entry->node;
	}

	action saw_comment {
		if (!list_empty(&list_stack))
			fnext list;

		curline++;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @saw_comment ->start ),

		# a table
		table: (
			name ']' @saw_table					->start	|
			'[' name ']' ']' @saw_table_array	->start
			),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any 	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]					-> basic_empty_or_multi_line	|
			[^"] ${fhold;}		-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]					-> basic_multi_line_start	|
			[^"] $saw_string	-> start
		),

		basic_multi_line_start: (
			'\n' ${curline++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${curline++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${curline++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${curline++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${curline++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'x'							-> hex_byte	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0xuU] ${*strp++=fc;}	${fret;}
		),

		hex_byte: (
			xdigit{2} >{hex[0]=*p;}
				@{hex[1]=*p; *strp++=strtoul(hex, NULL, 16); fret;}
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${curline++;}	-> literal_multi_line |
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${curline++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string								-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}		-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true				|
			'false'	@{number = 0;}				-> false			|
			'"'		${strp = string;}			-> basic_string	|
			[']		${strp = string;}			-> literal_string	|
			('-'|'+') ${fhold;number = 0;}		-> sign				|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->list		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->table		|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	char hex[3] = { 0 };
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, curline, p);
		free(parse_error);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_TABLE);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	CASE_ENUM_TO_STR(TOML_TABLE_ARRAY);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'|'['))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, curline);
			fbreak;
		}
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_table->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string + 1;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_table {
		char *ancestor, *tofree, *tablename;
		int item_added = 0;

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", (int)(p-ts), ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		char *ancestor, *tofree, *tablename;

		struct toml_node *place = toml_root;
		struct toml_list_item *new_table_entry = NULL;

		tofree = tablename = strndup(ts, (int)(p-ts-1));

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */

			/*
			 * Create a table array node and insert it into the heirarchy
			 */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE_ARRAY;
			list_head_init(&item->node.value.list);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		if (place->type != TOML_TABLE_ARRAY) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s",
															(int)(p-ts), ts);
			fbreak;
		}

		free(tofree);

		/*
		 * Create a table which becomes the last element in the list
		 * of maps (table array is a list of maps)
		 */
		new_table_entry = malloc(sizeof(*new_table_entry));
		if (!new_table_entry) {
			malloc_error = 1;
			fbreak;
		}

		new_table_entry->node.type = TOML_TABLE;
		new_table_entry->node.name = NULL;
		list_head_init(&new_table_entry->node.value.map);
		list_add_tail(&place->value.list, &new_table_entry->list);

		cur_table = &new_table_entry->node;
	}

	action saw_comment {
		if (!list_empty(&list_stack))
			fnext list;

		curline++;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @saw_comment ->start ),

		# a table
		table: (
			name ']' @saw_table					->start	|
			'[' name ']' ']' @saw_table_array	->start
			),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any 	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]					-> basic_empty_or_multi_line	|
			[^"] ${fhold;}		-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]					-> basic_multi_line_start	|
			[^"] $saw_string	-> start
		),

		basic_multi_line_start: (
			'\n' ${curline++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${curline++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${curline++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${curline++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${curline++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'x'							-> hex_byte	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0xuU] ${*strp++=fc;}	${fret;}
		),

		hex_byte: (
			xdigit{2} >{hex[0]=*p;}
				@{hex[1]=*p; *strp++=strtoul(hex, NULL, 16); fret;}
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${curline++;}	-> literal_multi_line |
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${curline++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string								-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}		-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true				|
			'false'	@{number = 0;}				-> false			|
			'"'		${strp = string;}			-> basic_string	|
			[']		${strp = string;}			-> literal_string	|
			('-'|'+') ${fhold;number = 0;}		-> sign				|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->list		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->table		|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	char hex[3] = { 0 };
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, curline, p);
		free(parse_error);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}


Ragel: rename

Ragel: rename

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit.call("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit.call("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}"
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| VcfHeader::RagelKeyValues.run_lexer(s) }

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit.call("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit.call("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}"
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| VcfHeader::RagelKeyValues.run_lexer(s) }

end


Fix game termination marker parsing for "*".

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static Result parse_game_termination_marker(Token *t)
{
	if (t->type == ASTERISK)
		return OTHER;
	if (t->type != SYMBOL)
		return NULL_RESULT;

	char *symbol = t->value.string;
	if (strcmp(symbol, "1-0") == 0)
		return WHITE_WINS;
	if (strcmp(symbol, "0-1") == 0)
		return BLACK_WINS;
	if (strcmp(symbol, "1/2-1/2") == 0)
		return DRAW;

	return NULL_RESULT;
}

static char *game_termination_marker(Result r)
{
	switch (r) {
	case WHITE_WINS: return "1-0";
	case BLACK_WINS: return "0-1";
	case DRAW: return "1/2-1/2";
	default: return "*";
	}
}

static bool parse_tokens(PGN *pgn, GArray *tokens, GError **err)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL) {
			if (tag_name_token->type == INTEGER) {
				g_set_error(err, 0, 0,
						"Unexpected token: %d", tag_name_token->value.integer);
			} else {
				g_set_error(err, 0, 0,
						"Unexpected token: %s", tag_name_token->value.string);
			}

			return false;
		}

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name)) {
			g_set_error(err, 0, 0, "Duplicate tag: %s", tag_name);
			return false;
		}

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING) {
			if (tag_name_token->type == INTEGER) {
				g_set_error(err, 0, 0, "Tag values must be strings, %d is not",
						tag_value_token->value.integer);
			} else {
				g_set_error(err, 0, 0, "Tag values must be strings, %s is not",
						tag_value_token->value.string);
			}

			return false;
		}

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET) {
			g_set_error(err, 0, 0,
					"Tag %s has no matching close bracket", tag_name);
			return false;
		}

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2) {
				g_set_error(err, 0, 0,
						"Incorrect move number %d (should be %d)",
						t->value.integer, half_move_number / 2);
				return false;
			}

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL && t->type != ASTERISK) {
			if (t->type == INTEGER) {
				g_set_error(err, 0, 0, "Expected a move, got %d",
						t->value.integer);
			} else {
				g_set_error(err, 0, 0, "Expected a move, got %s",
						t->value.string);
			}

			return false;
		}

		Result r;
		if ((r = parse_game_termination_marker(t)) != NULL_RESULT) {
			pgn->result = r;

			// If we didn't see a result tag, try to fill it in with the value
			// in the game termination marker

			if (!g_hash_table_contains(pgn->tags, "Result")) {
				char *tmp = malloc(strlen(t->value.string));
				strcpy(tmp, t->value.string);
				g_hash_table_insert(pgn->tags, "Result", tmp);
			}

			return true;
		}

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE) {
			g_set_error(err, 0, 0, "Expected a move, got %s", t->value.string);

			return false;
		}

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens, error);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

const char *seven_tag_roster[] =
{
	"Event", "Site", "Date", "Round", "White", "Black", "Result"
};

bool in_seven_tag_roster(char *tag_name)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++)
		if (strcmp(tag_name, seven_tag_roster[i]) == 0)
			return true;

	return false;
}

static void write_tag(FILE *file, const char *tag_name, const char *tag_value)
{
	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

static char *default_tag_value(const char *tag_name)
{
	if (strcmp(tag_name, "Date") == 0)
		return "????.??.??";
	if (strcmp(tag_name, "Result") == 0)
		return "*";
	else
		return "?";
}

static void process_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// We print the standard seven tags first, in a specified order, so don't
	// print them again.
	if (in_seven_tag_roster(tag_name))
		return;

	write_tag(file, tag_name, tag_value);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++) {
		const char *tag_name = seven_tag_roster[i];
		const char *tag_value = g_hash_table_contains(pgn->tags, tag_name) ?
			g_hash_table_lookup(pgn->tags, tag_name) :
			default_tag_value(tag_name);
		write_tag(file, tag_name, tag_value);
	}
	g_hash_table_foreach(pgn->tags, process_tag, (void *)file);

	putc('\n', file);

	Game *game = pgn->game->children;
	do {
		// A lot of the logic in here seems reversed/offset by one. This is
		// because the board associated with a particular move actually lives
		// at the parent node for that board
		if (game->board->turn == BLACK)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	fprintf(file, " %s\n", game_termination_marker(pgn->result));

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static Result parse_game_termination_marker(char *symbol)
{
	if (strcmp(symbol, "1-0") == 0)
		return WHITE_WINS;
	if (strcmp(symbol, "0-1") == 0)
		return BLACK_WINS;
	if (strcmp(symbol, "1/2-1/2") == 0)
		return DRAW;
	if (strcmp(symbol, "*") == 0)
		return OTHER;

	return NULL_MOVE;
}

static char *game_termination_marker(Result r)
{
	switch (r) {
	case WHITE_WINS: return "1-0";
	case BLACK_WINS: return "0-1";
	case DRAW: return "1/2-1/2";
	default: return "*";
	}
}

static bool parse_tokens(PGN *pgn, GArray *tokens, GError **err)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL) {
			if (tag_name_token->type == INTEGER) {
				g_set_error(err, 0, 0,
						"Unexpected token: %d", tag_name_token->value.integer);
			} else {
				g_set_error(err, 0, 0,
						"Unexpected token: %s", tag_name_token->value.string);
			}

			return false;
		}

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name)) {
			g_set_error(err, 0, 0, "Duplicate tag: %s", tag_name);
			return false;
		}

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING) {
			if (tag_name_token->type == INTEGER) {
				g_set_error(err, 0, 0, "Tag values must be strings, %d is not",
						tag_value_token->value.integer);
			} else {
				g_set_error(err, 0, 0, "Tag values must be strings, %s is not",
						tag_value_token->value.string);
			}

			return false;
		}

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET) {
			g_set_error(err, 0, 0,
					"Tag %s has no matching close bracket", tag_name);
			return false;
		}

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2) {
				g_set_error(err, 0, 0,
						"Incorrect move number %d (should be %d)",
						t->value.integer, half_move_number / 2);
				return false;
			}

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			if (t->type == INTEGER) {
				g_set_error(err, 0, 0, "Expected a move, got %d",
						t->value.integer);
			} else {
				g_set_error(err, 0, 0, "Expected a move, got %s",
						t->value.string);
			}

			return false;
		}

		Result r;
		if ((r = parse_game_termination_marker(t->value.string)) != NULL_RESULT) {
			pgn->result = r;

			// If we didn't see a result tag, try to fill it in with the value
			// in the game termination marker

			if (!g_hash_table_contains(pgn->tags, "Result")) {
				char *tmp = malloc(strlen(t->value.string));
				strcpy(tmp, t->value.string);
				g_hash_table_insert(pgn->tags, "Result", tmp);
			}

			return true;
		}

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE) {
			g_set_error(err, 0, 0, "Expected a move, got %s", t->value.string);

			return false;
		}

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens, error);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

const char *seven_tag_roster[] =
{
	"Event", "Site", "Date", "Round", "White", "Black", "Result"
};

bool in_seven_tag_roster(char *tag_name)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++)
		if (strcmp(tag_name, seven_tag_roster[i]) == 0)
			return true;

	return false;
}

static void write_tag(FILE *file, const char *tag_name, const char *tag_value)
{
	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

static char *default_tag_value(const char *tag_name)
{
	if (strcmp(tag_name, "Date") == 0)
		return "????.??.??";
	if (strcmp(tag_name, "Result") == 0)
		return "*";
	else
		return "?";
}

static void process_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// We print the standard seven tags first, in a specified order, so don't
	// print them again.
	if (in_seven_tag_roster(tag_name))
		return;

	write_tag(file, tag_name, tag_value);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++) {
		const char *tag_name = seven_tag_roster[i];
		const char *tag_value = g_hash_table_contains(pgn->tags, tag_name) ?
			g_hash_table_lookup(pgn->tags, tag_name) :
			default_tag_value(tag_name);
		write_tag(file, tag_name, tag_value);
	}
	g_hash_table_foreach(pgn->tags, process_tag, (void *)file);

	putc('\n', file);

	Game *game = pgn->game->children;
	do {
		// A lot of the logic in here seems reversed/offset by one. This is
		// because the board associated with a particular move actually lives
		// at the parent node for that board
		if (game->board->turn == BLACK)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	fprintf(file, " %s\n", game_termination_marker(pgn->result));

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}


[Task #214] do not introduce whitespace when executing mtocpp_post

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute' . (' '?)) => {};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('<' . [^&]* . '>') => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute'. (' '?)) => {};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ('<' . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] filename" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output." << endl;
}

int main(int argc, char ** argv)
{
  bool quiet = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet)
    cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir, quiet);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('<' . [^&]* . '>') => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] filename" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output." << endl;
}

int main(int argc, char ** argv)
{
  bool quiet = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet)
    cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir, quiet);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


remove redundant entry-guarded-concatenation operator

%%{
  machine hext;

  action error {
    throw_error();
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  builtin_name = ( alpha (alnum | '-' | '_')** );
  cap_var = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex = ( ( [^/] | '\\/' )** );
  match_regex = ( ( [^/] | '\\/' )** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          (
            '@'
            (
              builtin_name >{ LX_TK_START(TK_BUILTIN_NAME); }
                           %{ LX_TK_STOP; }
            )
          )
          |
          (
            attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                      %{ LX_TK_STOP; }
          )
        )
        (
          '='
          (
            (
              '{'
              (
                cap_var >{ LX_TK_START(TK_CAP_VAR); }
                        %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START(TK_MATCH_REGEX); }
                              %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                '"'
                (
                  match_literal >{ LX_TK_START(TK_MATCH_LITERAL); }
                                %{ LX_TK_STOP; }
                )
                '"'
              )
            )
          )
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          nth_child >{ LX_TK_START(TK_NTH_CHILD); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )?
      ):>
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    throw_error();
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  builtin_name = ( alpha (alnum | '-' | '_')** );
  cap_var = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex = ( ( [^/] | '\\/' )** );
  match_regex = ( ( [^/] | '\\/' )** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          (
            '@'
            (
              builtin_name >{ LX_TK_START(TK_BUILTIN_NAME); }
                           %{ LX_TK_STOP; }
            )
          )
          |
          (
            attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                      %{ LX_TK_STOP; }
          )
        )
        <:
        (
          '='
          (
            (
              '{'
              (
                cap_var >{ LX_TK_START(TK_CAP_VAR); }
                        %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START(TK_MATCH_REGEX); }
                              %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                '"'
                (
                  match_literal >{ LX_TK_START(TK_MATCH_LITERAL); }
                                %{ LX_TK_STOP; }
                )
                '"'
              )
            )
          )
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          nth_child >{ LX_TK_START(TK_NTH_CHILD); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )?
      ):>
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%


Add proper check to ragel email program

Add proper check to ragel email program

#include "../common.h"

/* From the Ragel manual:
        "To guard against this kind of problem one must ensure that the machine
         specification is divided up using boundaries that do not allow
         ambiguities from one portion of the machine to the next."
*/

char *mark;

%%{
  machine email_fst;
  action mark {
         mark = p;
  }
  action print {
    fprintf(stdout, "%s", mark);
  }

  estr = /[a-z0-9!#$%&'*+/=?^_`{|}~\-]/;
  beforeAt = estr+ ('.' estr+)*;
  az09 = /[a-z0-9]/;
  az09dash = /[a-z0-9\-]*[a-z0-9]/;
  part = az09 az09dash? '.';
  afterAt = part+ az09 az09dash?;

  email = beforeAt '@' afterAt;

  fb = /[^\n]*/;

  main := ((email >mark %print | (fb - email)) '\n')* ;
}%%
/*
Call "mark" upon entry to the sub-machine "email".
Call "print" when exiting the sub-maching "email".
*/
%% write data;

int main(int argc, char **argv) {

  PRE

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;

    if (p != pe) {
      FAIL;
    }
  }

  POST

  return 0;
}

#include "../common.h"

/* From the Ragel manual:
        "To guard against this kind of problem one must ensure that the machine
         specification is divided up using boundaries that do not allow
         ambiguities from one portion of the machine to the next."
*/

char *mark;

%%{
  machine email_fst;
  action mark {
         mark = p;
  }
  action print {
    fprintf(stdout, "%s", mark);
  }

  estr = /[a-z0-9!#$%&'*+/=?^_`{|}~\-]/;
  beforeAt = estr+ ('.' estr+)*;
  az09 = /[a-z0-9]/;
  az09dash = /[a-z0-9\-]*[a-z0-9]/;
  part = az09 az09dash? '.';
  afterAt = part+ az09 az09dash?;

  email = beforeAt '@' afterAt;

  fb = /[^\n]*/;

  main := ((email > 10 >mark %print | (fb > 5)) '\n')* ;
}%%
/*
Call "mark" upon entry to the sub-machine "email".
Call "print" when exiting the sub-maching "email".
*/
%% write data;

int main(int argc, char **argv) {

  PRE

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;
  }

  POST

  return 0;
}


remove optional space prefix from comments

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  comment = ( '#' (any - '\n')* '\n' );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  comment = ( ' '* '#' (any - '\n')* '\n' );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%