Fix move number printing for PGNs.

Fix move number printing for PGNs.

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static bool is_game_termination_marker(char *symbol)
{
	return strcmp(symbol, "1-0") == 0 ||
		strcmp(symbol, "0-1") == 0 ||
		strcmp(symbol, "1/2-1/2") == 0;
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;
		if (is_game_termination_marker(t->value.string))
			return true;

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

static void write_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	// TODO: sorting
	g_hash_table_foreach(pgn->tags, write_tag, (void *)file);

	Game *game = pgn->game->children;
	do {
		// A lot of the logic in here seems reversed/offset by one. This is
		// because the board associated with a particular move actually lives
		// at the parent node for that board
		if (game->board->turn == BLACK)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	// TODO: write game termination marker - requires storing the result in
	// the Game struct

	putc('\n', file);

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static bool is_game_termination_marker(char *symbol)
{
	return strcmp(symbol, "1-0") == 0 ||
		strcmp(symbol, "0-1") == 0 ||
		strcmp(symbol, "1/2-1/2") == 0;
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;
		if (is_game_termination_marker(t->value.string))
			return true;

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

static void write_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	// TODO: sorting
	g_hash_table_foreach(pgn->tags, write_tag, (void *)file);

	Game *game = pgn->game->children;
	do {
		if (game->board->turn == WHITE)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		// We use game->parent->board here, as that is the baord in which
		// this move was made.
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	// TODO: write game termination marker - requires storing the result in
	// the Game struct

	putc('\n', file);

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}


improve bulk string parsing performance

improve bulk string parsing performance

// -*- coding: utf-8 -*-
%%{

machine resp;

action mark {
  mark = fpc;
}

action simple_reply {
  String delimitedValue = new String(Arrays.copyOfRange(data, mark, fpc - 1));
  emit(new SimpleString(delimitedValue));
}

action error_reply {
  String delimitedValue = new String(Arrays.copyOfRange(data, mark, fpc - 1));
  emit(new Error(delimitedValue));
}

action integer_reply {
  emit(Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc - 1))));
}

action empty_bulk_reply {
  emit("");
}

action null_bulk_reply {
  emit(null);
}

action bulk_length {
  bulkLength = (int)Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc + 1)));
  minimumBufferSize = Math.max(fpc + 2 + bulkLength + 2, pe);
}

action start_bulk {
  mark = fpc;
}

action bulk_skip {
  if ((fpc - mark) == (bulkLength - 1)) {
    fnext resp_bulk_end;
  } else {
    fexec Math.min(fpc + bulkLength - 2, pe);
  }
}

action bulk_reply {
  emit(Arrays.copyOfRange(data, mark, mark + bulkLength));
}

action push_empty_array {
  emit(new Array(0, currentParent));
}

action push_null_array {
  emit(null);
}

action push_array {
  int len = (int)Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc - 1)));
  Array arr = new Array(len, currentParent);
  emit(arr);
}

action check_if_reply_complete {
  if ((fpc + 1) == pe && isDone()) {
    parseState = PARSE_COMPLETE;
  } else if ((fpc + 1) < pe) {
    if (isDone()) {
      parseState = PARSE_OVERFLOW;
      fbreak;
    } else {
      fgoto reply;
    }
  }
}

crlf                        = "\r\n";

# RESP Simple
resp_simple                 := "+" (any* -- crlf) >mark crlf @simple_reply @check_if_reply_complete;

# RESP Error
resp_error                  := "-" (any* -- crlf) >mark crlf @error_reply @check_if_reply_complete;

# RESP Integers
resp_integer                := ":" ("-"? digit+) >mark crlf @integer_reply @check_if_reply_complete;

# RESP Bulk Strings
resp_bulk_nil               = "-1" crlf @null_bulk_reply;
resp_bulk_empty             = "0" crlf{2} @empty_bulk_reply;
resp_bulk_end               = crlf; # used for fnext
resp_bulk_content           = ([1-9] digit*) >mark @bulk_length crlf any >start_bulk (any >bulk_skip)* resp_bulk_end @bulk_reply;
resp_bulk                   := "$" (resp_bulk_empty | resp_bulk_nil | resp_bulk_content) @check_if_reply_complete;

# RESP Arrays
resp_null_array             = "-1" crlf;
resp_empty_array            = "0" crlf;
resp_non_empty_array_header = ([1-9] digit*) >mark crlf @push_array;
resp_non_empty_array        = resp_non_empty_array_header @{ fgoto reply; };
resp_array                  := "*" (resp_null_array @push_null_array | resp_empty_array @push_empty_array | resp_non_empty_array_header) @check_if_reply_complete;

reply                       := ('+' @{ fhold; fgoto resp_simple;   } |
                                '-' @{ fhold; fgoto resp_error;    } |
                                ':' @{ fhold; fgoto resp_integer;  } |
                                '$' @{ fhold; fgoto resp_bulk;     } |
                                '*' @{ fhold; fgoto resp_array;    });

}%%

package redis.resp;

import java.util.*;
import java.util.logging.Logger;
import redis.resp.SimpleString;
import redis.resp.Array;
import redis.resp.Error;
import java.nio.file.*;

public class ReplyParser {
  private static Logger LOG = Logger.getLogger("redis.resp.ReplyParser");

  // heavily inspired by this great tutorial
  // http://www.avrfreaks.net/forum/codec-parsing-json-based-config-file-using-micro-memory-ragel
  public static final int PARSE_NOT_STARTED = -1;
  public static final int PARSE_ERROR       = -2;
  public static final int PARSE_OK          = 0;
  public static final int PARSE_INCOMPLETE  = 1;
  public static final int PARSE_COMPLETE    = 2;
  public static final int PARSE_OVERFLOW    = 3;

  public static final int SIMPLE_STRING_REPLY  = 0;
  public static final int ERROR_REPLY          = 1;

  public static final Array EMPTY_ARRAY = new Array(0);

  // parser state
  public final Array root;
  private Array currentParent = null;
  private int mark            = 0;
  private int bulkLength      = -1;
  private int bulkConsumed    = 0;

  private int parseState         = PARSE_NOT_STARTED;
  private byte[] data            = null;
  private int delimitedReplyType = -1;

  // buffer hint
  private int minimumBufferSize = -1;

  // ragel members
  private int p;
  private int eof;
  private int pe;
  private int te;
  private int ts;
  private int cs;
  private int act;

  %% write data;

  public ReplyParser() {
    this(1);
  }

  public ReplyParser(int replies) {
    %% write init;
    root = new Array(replies);
    currentParent = root;
  }

  public static void println(String s) {
    //System.out.println(s);
  }

  public static void print(String s) {
    //System.out.print(s);
  }

  public void printchars(byte[] data, int start, int stop) {
    if (!"1".equals(System.getProperty("redis.resp.debug"))) {
      return;
    }

    print("|");
    for (int i = 0; i < stop - start; i++) {
      char ch = (char)data[start + i];
      String s = null;

      switch(ch) {
        case '\r': s = "\\r"; break;
        case '\n': s = "\\n"; break;
        default:
          s = " " + Character.toString(ch);
      }

      print(" " + s + " |");
    }

    print("\n");

    for (int i = 0; i < stop - start; i++) {
      String s = "  ";
      if (mark == (start + i)) {
        s = "  ^";
      } else if (ts == (start + i)) {
        s = "  *";
      }
      print(" " + s + " ");
    }

    print("\n");
  }

  private void unwind() {
    while (currentParent != null && currentParent.isFull()) {
      currentParent = currentParent.parent;
    }
  }

  public boolean isDone() {
    return currentParent == null && root.isComplete();
  }

  public void emit(Object value) {
    currentParent.addItem(value);
    unwind();
  }

  public void emit(Array newMultiBulk) {
    currentParent.addItem(newMultiBulk);

    if (newMultiBulk != null) {
      currentParent = newMultiBulk;
    }

    unwind();
  }

  public byte[] getOverflow() {
    byte[] overflow = new byte[pe - p];
    System.arraycopy(data, p, overflow, 0, overflow.length);
    return overflow;
  }

  public Object parse(byte[] buffer) {
    if (data != null) {
      int available = data.length - pe;

      // check for room in the current buffer
      if (available >= buffer.length) {

          System.arraycopy(buffer, 0, data, pe, buffer.length);
          eof += buffer.length;
          pe  += buffer.length;

      } else {

          byte[] previousBuffer = data;
          int previousDataLength = pe;
          int dataLength = previousDataLength + buffer.length;

          int newBufferSize = Math.max(dataLength, minimumBufferSize);

          data = new byte[newBufferSize];

          System.arraycopy(previousBuffer, 0, data, 0, previousDataLength);
          System.arraycopy(buffer, 0, data, previousDataLength, buffer.length);

          eof = dataLength;
          pe  = dataLength;
      }

    } else {

      data              = buffer;
      p                 = 0;
      eof               = data.length;
      pe                = data.length;
      parseState        = PARSE_INCOMPLETE;
      minimumBufferSize = data.length;
    }

    %% write exec;

    return parseState;
  }

  public Object parse(String data) {
    return parse(data.getBytes());
  }

  public static void main(String[] args) throws Exception {
    ReplyParser parser = new ReplyParser(1);

    int parseState = -1;
    println("=====");
    for (int i = 0; i < args.length; i++) {
        println("Input: " + args[i]);
        parseState = (int)parser.parse(args[i]);
        println("" + parseState);
    }

    if (parseState == 3) {
      println("Overflow:");
      byte[] overflow = parser.getOverflow();
      parser.printchars(overflow, 0, overflow.length);
    }
  }
}

// -*- coding: utf-8 -*-
%%{

machine resp;

action mark {
  mark = fpc;
}

action simple_reply {
  String delimitedValue = new String(Arrays.copyOfRange(data, mark, fpc - 1));
  emit(new SimpleString(delimitedValue));
}

action error_reply {
  String delimitedValue = new String(Arrays.copyOfRange(data, mark, fpc - 1));
  emit(new Error(delimitedValue));
}

action integer_reply {
  emit(Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc - 1))));
}

action empty_bulk_reply {
  emit("");
}

action null_bulk_reply {
  emit(null);
}

action bulk_length {
  bulkLength = (int)Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc + 1)));

  // hint the minimum size buffer needed to satisfy everything up-to the current bulk string for later incremental parsing
  minimumBufferSize = Math.max(fpc + 2 + bulkLength + 2, pe);
}

action start_bulk {
  mark         = fpc;
  bulkConsumed = 0;
}

action has_bulk_consumed {
  ++bulkConsumed < bulkLength
}

action bulk_reply {
  emit(Arrays.copyOfRange(data, mark, mark + bulkLength));
}

action push_empty_array {
  emit(new Array(0, currentParent));
}

action push_null_array {
  emit(null);
}

action push_array {
  int len = (int)Long.parseLong(new String(Arrays.copyOfRange(data, mark, fpc - 1)));
  Array arr = new Array(len, currentParent);
  emit(arr);
}

action check_if_reply_complete {
  if ((fpc + 1) == pe && isDone()) {
    parseState = PARSE_COMPLETE;
  } else if ((fpc + 1) < pe) {
    if (isDone()) {
      parseState = PARSE_OVERFLOW;
      fbreak;
    } else {
      fgoto reply;
    }
  }
}

action debug {
  println("debug: fpc: " + fpc + ", pe: " + pe);
  printchars(data, fpc, pe);
  println(root.toString());
}

crlf                        = "\r\n";

# RESP Simple
resp_simple                 := "+" simple_string: (any* -- crlf) >mark crlf @simple_reply @check_if_reply_complete;

# RESP Error
resp_error                  := "-" (any* -- crlf) >mark crlf @error_reply @check_if_reply_complete;

# RESP Integers
resp_integer                := ":" ("-"? digit+) >mark crlf @integer_reply @check_if_reply_complete;

# RESP Bulk Strings
resp_bulk_nil               = "-1" crlf @null_bulk_reply;
resp_bulk_empty             = "0" crlf{2} @empty_bulk_reply;
resp_bulk_content           = ([1-9] digit*) >mark @bulk_length crlf any >start_bulk (any when has_bulk_consumed)* crlf @bulk_reply;
resp_bulk                   := "$" (resp_bulk_empty | resp_bulk_nil | resp_bulk_content) @check_if_reply_complete;

# RESP Arrays
resp_null_array             = "-1" crlf;
resp_empty_array            = "0" crlf;
resp_non_empty_array_header = ([1-9] digit*) >mark crlf @push_array;
resp_non_empty_array        = resp_non_empty_array_header @{ fgoto reply; };
resp_array                  := "*" (resp_null_array @push_null_array | resp_empty_array @push_empty_array | resp_non_empty_array_header) @check_if_reply_complete;

reply                       := ('+' @{ fhold; fgoto resp_simple;   } |
                                '-' @{ fhold; fgoto resp_error;    } |
                                ':' @{ fhold; fgoto resp_integer;  } |
                                '$' @{ fhold; fgoto resp_bulk;     } |
                                '*' @{ fhold; fgoto resp_array;    });

}%%

package redis.resp;

import java.util.*;
import java.util.logging.Logger;
import redis.resp.SimpleString;
import redis.resp.Array;
import redis.resp.Error;
import java.nio.file.*;

public class ReplyParser {
  private static Logger LOG = Logger.getLogger("redis.resp.ReplyParser");

  // heavily inspired by this great tutorial
  // http://www.avrfreaks.net/forum/codec-parsing-json-based-config-file-using-micro-memory-ragel
  public static final int PARSE_NOT_STARTED = -1;
  public static final int PARSE_ERROR       = -2;
  public static final int PARSE_OK          = 0;
  public static final int PARSE_INCOMPLETE  = 1;
  public static final int PARSE_COMPLETE    = 2;
  public static final int PARSE_OVERFLOW    = 3;

  public static final int SIMPLE_STRING_REPLY  = 0;
  public static final int ERROR_REPLY          = 1;

  public static final Array EMPTY_ARRAY = new Array(0);

  // parser state
  public final Array root;
  private Array currentParent = null;
  private int mark            = 0;
  private int bulkLength      = -1;
  private int bulkConsumed    = 0;

  private int parseState         = PARSE_NOT_STARTED;
  private byte[] data            = null;
  private int delimitedReplyType = -1;

  // buffer hint
  private int minimumBufferSize = -1;

  // ragel members
  private int p;
  private int eof;
  private int pe;
  private int te;
  private int ts;
  private int cs;
  private int act;

  %% write data;

  public ReplyParser() {
    this(1);
  }

  public ReplyParser(int replies) {
    %% write init;
    root = new Array(replies);
    currentParent = root;
  }

  public static void println(String s) {
    //System.out.println(s);
  }

  public static void print(String s) {
    System.out.print(s);
  }

  public void printchars(byte[] data, int start, int stop) {
    if (!"1".equals(System.getProperty("redis.resp.debug"))) {
      return;
    }

    print("|");
    for (int i = 0; i < stop - start; i++) {
      char ch = (char)data[start + i];
      String s = null;

      switch(ch) {
        case '\r': s = "\\r"; break;
        case '\n': s = "\\n"; break;
        default:
          s = " " + Character.toString(ch);
      }

      print(" " + s + " |");
    }

    print("\n");

    for (int i = 0; i < stop - start; i++) {
      String s = "  ";
      if (mark == (start + i)) {
        s = "  ^";
      } else if (ts == (start + i)) {
        s = "  *";
      }
      print(" " + s + " ");
    }

    print("\n");
  }

  private void unwind() {
    while (currentParent != null && currentParent.isFull()) {
      currentParent = currentParent.parent;
    }
  }

  public boolean isDone() {
    return currentParent == null && root.isComplete();
  }

  public void emit(Object value) {
    currentParent.addItem(value);
    unwind();
  }

  public void emit(Array newMultiBulk) {
    currentParent.addItem(newMultiBulk);

    if (newMultiBulk != null) {
      currentParent = newMultiBulk;
    }

    unwind();
  }

  public byte[] getOverflow() {
    byte[] overflow = new byte[pe - p];
    System.arraycopy(data, p, overflow, 0, overflow.length);
    return overflow;
  }

  public Object parse(byte[] buffer) {
    if (data != null) {
      int available = data.length - pe;

      // check for room in the current buffer
      if (available >= buffer.length) {

          System.arraycopy(buffer, 0, data, pe, buffer.length);
          eof += buffer.length;
          pe  += buffer.length;

      } else {

          byte[] previousBuffer = data;
          int previousDataLength = pe;
          int dataLength = previousDataLength + buffer.length;

          int newBufferSize = Math.max(dataLength, minimumBufferSize);

          data = new byte[newBufferSize];

          System.arraycopy(previousBuffer, 0, data, 0, previousDataLength);
          System.arraycopy(buffer, 0, data, previousDataLength, buffer.length);

          eof = dataLength;
          pe  = dataLength;
      }

    } else {

      data              = buffer;
      p                 = 0;
      eof               = data.length;
      pe                = data.length;
      parseState        = PARSE_INCOMPLETE;
      minimumBufferSize = data.length;
    }

    %% write exec;

    return parseState;
  }

  public Object parse(String data) {
    return parse(data.getBytes());
  }

  public static void main(String[] args) throws Exception {
    ReplyParser parser = new ReplyParser(1);

    int parseState = -1;
    println("=====");
    for (int i = 0; i < args.length; i++) {
        println("Input: " + args[i]);
        parseState = (int)parser.parse(args[i]);
        println("" + parseState);
    }

    if (parseState == 3) {
      println("Overflow:");
      byte[] overflow = parser.getOverflow();
      parser.printchars(overflow, 0, overflow.length);
    }
  }
}


Move input scanning into a separate method

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  unsigned_integer = digit+;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    'E'              { emit('E');           };

    unsigned_integer { emit_unsigned_integer(); };
    identifier       { emit_identifier();       };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_identifier
      identifier = text
      identifier.gsub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def text
      @buffer[@ts...@te].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  unsigned_integer = digit+;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    'E'              { emit('E');           };

    unsigned_integer { emit_unsigned_integer(); };
    identifier       { emit_identifier();       };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block

      # Tokenize a stream of input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_identifier
      identifier = text
      identifier.gsub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def text
      @buffer[@ts...@te].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL


Updated state machine that fixes the duplicate registrations problem.

/**
 *
 * Copyright (c) 2010, Zed A. Shaw and Mongrel2 Project Contributors.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 *     * Neither the name of the Mongrel2 Project, Zed A. Shaw, nor the names
 *       of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

%%{
    machine State;

    import "events.h";

Proxy := (
        start: (
           CONNECT @proxy_deliver -> Sending |
           FAILED @proxy_failed -> Closing
        ),

        Proxying: (
            HTTP_REQ @proxy_deliver -> Sending |
            PROXY @proxy_exit_routing |
            HANDLER @proxy_exit_routing |
            DIRECTORY @proxy_exit_routing |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Sending: (
            REQ_SENT @proxy_reply_parse -> Receiving |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Receiving: (
            REQ_RECV @proxy_req_parse -> Proxying |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Closing: (
            CLOSE @proxy_exit_idle
        )

     )  <err(error);

Connection = (
        start: ( OPEN @open -> Accepting ),

        Accepting: ( ACCEPT @parse -> Identifying ),

        Identifying: (
            REQ_RECV @register_request -> Registered |
            CLOSE @close -> final
        ),

        Registered: (
            HTTP_REQ @route_request -> HTTPRouting |
            MSG_REQ @route_request -> MSGRouting |
            SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        Idle: (
            REQ_RECV @identify_request HTTP_REQ @route_request -> HTTPRouting |
            REQ_RECV @identify_request MSG_REQ @route_request -> MSGRouting |
            REQ_RECV @identify_request SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        MSGRouting: ( HANDLER @msg_to_handler -> Queueing ),

        HTTPRouting: (
            HANDLER @http_to_handler -> Queueing |
            PROXY @http_to_proxy  |
            DIRECTORY @http_to_directory -> Responding |
            CLOSE @close -> final
        ),

        Queueing: ( REQ_SENT @parse -> Idle ),

        Responding: (
            RESP_SENT @parse -> Idle |
            CLOSE @close -> final
        )

    ) %eof(finish) <err(error);

main := (Connection)*;

}%%

/**
 *
 * Copyright (c) 2010, Zed A. Shaw and Mongrel2 Project Contributors.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 *     * Neither the name of the Mongrel2 Project, Zed A. Shaw, nor the names
 *       of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

%%{
    machine State;

    import "events.h";

Proxy := (
        start: (
           CONNECT @proxy_deliver -> Sending |
           FAILED @proxy_failed -> Closing
        ),

        Proxying: (
            HTTP_REQ @proxy_deliver -> Sending |
            PROXY @proxy_exit_routing |
            HANDLER @proxy_exit_routing |
            DIRECTORY @proxy_exit_routing |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Sending: (
            REQ_SENT @proxy_reply_parse -> Receiving |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Receiving: (
            REQ_RECV @proxy_req_parse -> Proxying |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Closing: (
            CLOSE @proxy_exit_idle
        )

     )  <err(error);

Connection = (
        start: ( OPEN @open -> Accepting ),

        Accepting: ( ACCEPT @parse -> Idle ),

        Idle: (
            REQ_RECV @identify_request HTTP_REQ @route_request -> HTTPRouting |
            REQ_RECV @identify_request MSG_REQ @route_request -> MSGRouting |
            REQ_RECV @identify_request SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        MSGRouting: ( HANDLER @msg_to_handler -> Queueing ),

        HTTPRouting: (
            HANDLER @http_to_handler -> Queueing |
            PROXY @http_to_proxy  |
            DIRECTORY @http_to_directory -> Responding |
            CLOSE @close -> final
        ),

        Queueing: ( REQ_SENT @parse -> Idle ),

        Responding: (
            RESP_SENT @parse -> Idle |
            CLOSE @close -> final
        )

    ) %eof(finish) <err(error);

main := (Connection)*;

}%%


Accept more characters in identifiers

Accept more characters in identifiers

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;

		std::cerr
		<< "** Buffer contents from the start:" << std::endl;
		std::copy(m_buffer.cbegin(), m_buffer.cend(), std::ostream_iterator <char>(std::cerr));
		std::cerr << std::endl;

		std::string_view buffer_end(m_fsm.p, m_fsm.pe - m_fsm.p);
		std::cerr
		<< "** Buffer contents starting from m_fsm.p:" << std::endl
		<< buffer_end << std::endl;

		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
		m_fsm.eof = nullptr;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action set_unknown_quality_value {
				m_integer = std::numeric_limits <decltype(m_integer)>::max();
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if (!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chr			= graph;

			chrom_id	= (chr+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | (chr - ';'))+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual_numeric	= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };
			qual_unknown	= '.'
				$(set_unknown_quality_value)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			qual			= qual_numeric | qual_unknown;

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= (chr - ';')+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_str		= (chr - [,;=]) +;
			info_key		= info_str;
			info_val		= info_str (',' info_str)*;
			info_part		= info_key ('=' info_val)?;
			info_missing	= '.';
			info			= info_missing | (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;

		std::cerr
		<< "** Buffer contents from the start:" << std::endl;
		std::copy(m_buffer.cbegin(), m_buffer.cend(), std::ostream_iterator <char>(std::cerr));
		std::cerr << std::endl;

		std::string_view buffer_end(m_fsm.p, m_fsm.pe - m_fsm.p);
		std::cerr
		<< "** Buffer contents starting from m_fsm.p:" << std::endl
		<< buffer_end << std::endl;

		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
		m_fsm.eof = nullptr;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action set_unknown_quality_value {
				m_integer = std::numeric_limits <decltype(m_integer)>::max();
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if (!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | alnum)+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual_numeric	= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };
			qual_unknown	= '.'
				$(set_unknown_quality_value)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			qual			= qual_numeric | qual_unknown;

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key		= (alnum | '_') +;
			info_str		= (alnum | [+-_.]) +;
			info_val		= info_str (',' info_str)*;
			info_part		= info_key ('=' info_val)?;
			info_missing	= '.';
			info			= info_missing | (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}