added assign and removed ' as a symbol

added assign and removed ' as a symbol

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

%%{
    machine sast;

    whitespace      = ' '+;
    assign          = '=';
    tab             = '\t';
    hash            = '#';
    doublequote     = '"';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|','|'('|')'|'['|']'|'{'|'}'|':'|';';
    symbol          = (any -- ('#'|whitespace|newline|assign|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ('#' (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String, 1);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            Token(fc);
            ++nesting;
        };

        '}'|']'|')' => {
            Token(fc);
            --nesting;
        };

        number => {
            Token(TokenKind::Number);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class SotaLexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    int nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    SotaLexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~SotaLexer() {
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
        else
            printf("UNEXPECTED BEHAVIOR!!!\n");
    }

    const char * Newline(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Line(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos(const char *pchar) {
        const char *nl = Newline(pchar);
        if (nl)
            return pchar - nl;
        return 0;
    }

    void Token(long kind, long trim=0) {
        tokens.push_back({
            ts - source + trim,
            te - source - trim,
            kind,
            Line(ts),
            Pos(ts)});
    }

    void Token(long start, long end, long kind, long line, long pos, long skip) {
        tokens.push_back({
            start,
            end,
            kind,
            line,
            pos,
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    SotaLexer lexer(source);
    return lexer.Scan(tokens);
}

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

%%{
    machine sast;

    whitespace      = ' '+;
    tab             = '\t';
    hash            = '#';
    doublequote     = '"';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|','|'('|')'|'['|']'|'{'|'}'|':'|';';
    symbol          = (any - ('#'|whitespace|newline|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ('#' (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String, 1);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            Token(fc);
            ++nesting;
        };

        '}'|']'|')' => {
            Token(fc);
            --nesting;
        };

        number => {
            Token(TokenKind::Number);
        };

        "'" => {
            Token(TokenKind::Symbol);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class SotaLexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    int nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    SotaLexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~SotaLexer() {
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
        else
            printf("UNEXPECTED BEHAVIOR!!!\n");
    }

    const char * Newline(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Line(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos(const char *pchar) {
        const char *nl = Newline(pchar);
        if (nl)
            return pchar - nl;
        return 0;
    }

    void Token(long kind, long trim=0) {
        tokens.push_back({
            ts - source + trim,
            te - source - trim,
            kind,
            Line(ts),
            Pos(ts)});
    }

    void Token(long start, long end, long kind, long line, long pos, long skip) {
        tokens.push_back({
            start,
            end,
            kind,
            line,
            pos,
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    SotaLexer lexer(source);
    return lexer.Scan(tokens);
}


Newline in between tags and movetext.

Newline in between tags and movetext.

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static Result parse_game_termination_marker(char *symbol)
{
	if (strcmp(symbol, "1-0") == 0)
		return WHITE_WINS;
	if (strcmp(symbol, "0-1") == 0)
		return BLACK_WINS;
	if (strcmp(symbol, "1/2-1/2") == 0)
		return DRAW;
	if (strcmp(symbol, "*") == 0)
		return OTHER;

	return NULL_MOVE;
}

static char *game_termination_marker(Result r)
{
	switch (r) {
	case WHITE_WINS: return "1-0";
	case BLACK_WINS: return "0-1";
	case DRAW: return "1/2-1/2";
	default: return "*";
	}
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;

		Result r;
		if ((r = parse_game_termination_marker(t->value.string)) != NULL_RESULT) {
			pgn->result = r;

			// If we didn't see a result tag, try to fill it in with the value
			// in the game termination marker

			if (!g_hash_table_contains(pgn->tags, "Result")) {
				char *tmp = malloc(strlen(t->value.string));
				strcpy(tmp, t->value.string);
				g_hash_table_insert(pgn->tags, "Result", tmp);
			}

			return true;
		}

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

const char *seven_tag_roster[] =
{
	"Event", "Site", "Date", "Round", "White", "Black", "Result"
};

bool in_seven_tag_roster(char *tag_name)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++)
		if (strcmp(tag_name, seven_tag_roster[i]) == 0)
			return true;

	return false;
}

static void write_tag(FILE *file, const char *tag_name, const char *tag_value)
{
	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

static char *default_tag_value(const char *tag_name)
{
	if (strcmp(tag_name, "Date") == 0)
		return "????.??.??";
	if (strcmp(tag_name, "Result") == 0)
		return "*";
	else
		return "?";
}

static void process_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// We print the standard seven tags first, in a specified order, so don't
	// print them again.
	if (in_seven_tag_roster(tag_name))
		return;

	write_tag(file, tag_name, tag_value);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++) {
		const char *tag_name = seven_tag_roster[i];
		const char *tag_value = g_hash_table_contains(pgn->tags, tag_name) ?
			g_hash_table_lookup(pgn->tags, tag_name) :
			default_tag_value(tag_name);
		write_tag(file, tag_name, tag_value);
	}
	g_hash_table_foreach(pgn->tags, process_tag, (void *)file);

	putc('\n', file);

	Game *game = pgn->game->children;
	do {
		// A lot of the logic in here seems reversed/offset by one. This is
		// because the board associated with a particular move actually lives
		// at the parent node for that board
		if (game->board->turn == BLACK)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	fprintf(file, " %s\n", game_termination_marker(pgn->result));

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static Result parse_game_termination_marker(char *symbol)
{
	if (strcmp(symbol, "1-0") == 0)
		return WHITE_WINS;
	if (strcmp(symbol, "0-1") == 0)
		return BLACK_WINS;
	if (strcmp(symbol, "1/2-1/2") == 0)
		return DRAW;
	if (strcmp(symbol, "*") == 0)
		return OTHER;

	return NULL_MOVE;
}

static char *game_termination_marker(Result r)
{
	switch (r) {
	case WHITE_WINS: return "1-0";
	case BLACK_WINS: return "0-1";
	case DRAW: return "1/2-1/2";
	default: return "*";
	}
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;

		Result r;
		if ((r = parse_game_termination_marker(t->value.string)) != NULL_RESULT) {
			pgn->result = r;

			// If we didn't see a result tag, try to fill it in with the value
			// in the game termination marker

			if (!g_hash_table_contains(pgn->tags, "Result")) {
				char *tmp = malloc(strlen(t->value.string));
				strcpy(tmp, t->value.string);
				g_hash_table_insert(pgn->tags, "Result", tmp);
			}

			return true;
		}

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

const char *seven_tag_roster[] =
{
	"Event", "Site", "Date", "Round", "White", "Black", "Result"
};

bool in_seven_tag_roster(char *tag_name)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++)
		if (strcmp(tag_name, seven_tag_roster[i]) == 0)
			return true;

	return false;
}

static void write_tag(FILE *file, const char *tag_name, const char *tag_value)
{
	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

static char *default_tag_value(const char *tag_name)
{
	if (strcmp(tag_name, "Date") == 0)
		return "????.??.??";
	if (strcmp(tag_name, "Result") == 0)
		return "*";
	else
		return "?";
}

static void process_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// We print the standard seven tags first, in a specified order, so don't
	// print them again.
	if (in_seven_tag_roster(tag_name))
		return;

	write_tag(file, tag_name, tag_value);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	size_t size = sizeof(seven_tag_roster) / sizeof(seven_tag_roster[0]);
	for (size_t i = 0; i < size; i++) {
		const char *tag_name = seven_tag_roster[i];
		const char *tag_value = g_hash_table_contains(pgn->tags, tag_name) ?
			g_hash_table_lookup(pgn->tags, tag_name) :
			default_tag_value(tag_name);
		write_tag(file, tag_name, tag_value);
	}
	g_hash_table_foreach(pgn->tags, process_tag, (void *)file);

	Game *game = pgn->game->children;
	do {
		// A lot of the logic in here seems reversed/offset by one. This is
		// because the board associated with a particular move actually lives
		// at the parent node for that board
		if (game->board->turn == BLACK)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	fprintf(file, " %s\n", game_termination_marker(pgn->result));

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}


[Fix] Simplify state machine by ignoring multiple spaces

%%{
  machine newlines_strip;

  action Double_CRLF {
    if (!crlf_added && p > c) {
      (*newlines_count)++;
      g_byte_array_append (data, (const guint8 *)"\n", 1);
      c = p;
    }

    crlf_added = TRUE;
    c = p;
  }

  action WSP {
    g_byte_array_append (data, (const guint8 *)" ", 1);
    c = p;
  }

  action Text_Start {
    crlf_added = FALSE;
    c = p;
  }

  action Text_End {
    if (p > c) {
      g_byte_array_append (data, (const guint8 *)c, p - c);
      last_c = *(p - 1);
    }

    c = p;
  }

  action Line_CRLF {
    if (!crlf_added) {
      if (is_html || g_ascii_ispunct (last_c)) {
         g_byte_array_append (data, (const guint8 *)"\n", 1);
         crlf_added = TRUE;
      }
    }

    (*newlines_count)++;
    g_ptr_array_add (newlines, (gpointer)p);
    c = p;
  }

  WSP   = " " | "\t" | "\v";
  CRLF  = ("\r" . "\n") | ( "\r" ) | ("\n");
  DOUBLE_CRLF = (CRLF <: (WSP* CRLF)+) %Double_CRLF;
  ANY_CRLF = CRLF | DOUBLE_CRLF;
  LINE = (([^\r\n]+) >Text_Start %Text_End);
  TEXT  = ANY_CRLF* . (LINE <: ANY_CRLF %Line_CRLF)+ | LINE | ANY_CRLF %Line_CRLF;

  main := TEXT;
}%%

#include <glib.h>

%% write data;

void
rspamd_strip_newlines_parse (const gchar *begin, const gchar *pe,
    GByteArray *data, gboolean is_html, guint *newlines_count,
    GPtrArray *newlines)
{
  const gchar *c, *p, *eof;
  gint last_c = -1;
  gint cs = 0;
  gboolean crlf_added = FALSE;

  c = begin;
  p = begin;
  eof = pe;

  %% write init;
  %% write exec;

  if (p > c) {
     g_byte_array_append (data, (const guint8 *)c, p - c);
  }
}

%%{
  machine newlines_strip;

  action Double_CRLF {
    if (!crlf_added) {
      g_byte_array_append (data, (const guint8 *)"\n", 1);
      c = p;
    }

    crlf_added = TRUE;
    c = p;
  }

  action WSP {
    g_byte_array_append (data, (const guint8 *)" ", 1);
    c = p;
  }

  action Text_Start {
    crlf_added = FALSE;
    c = p;
  }

  action Text_End {
    if (p > c) {
      g_byte_array_append (data, (const guint8 *)c, p - c);
      last_c = *(p - 1);
    }

    c = p;
  }

  action Line_CRLF {
    if (!crlf_added) {
      if (is_html || g_ascii_ispunct (last_c)) {
         g_byte_array_append (data, (const guint8 *)"\n", 1);
         crlf_added = TRUE;
      }
    }

    (*newlines_count)++;
    g_ptr_array_add (newlines, (gpointer)p);
    c = p;
  }

  WSP   = " " | "\t" | "\v";
  CRLF  = ("\r" . "\n") | ( "\r" ) | ("\n");
  DOUBLE_CRLF = (CRLF <: (WSP* CRLF)+) %Double_CRLF;
  ANY_CRLF = CRLF | DOUBLE_CRLF;
  LINE_ELT = ((WSP+ %WSP)** :> ((^space)+) >Text_Start %Text_End <: (WSP+ %WSP)**);
  LINE = LINE_ELT+;
  TEXT  = ANY_CRLF** . (LINE <: ANY_CRLF %Line_CRLF)+ | LINE | ANY_CRLF %Line_CRLF;

  main := TEXT;
}%%

#include <glib.h>

%% write data;

void
rspamd_strip_newlines_parse (const gchar *begin, const gchar *pe,
    GByteArray *data, gboolean is_html, guint *newlines_count,
    GPtrArray *newlines)
{
  const gchar *c, *p, *eof;
  gint last_c = -1;
  gint cs = 0;
  gboolean crlf_added = FALSE;

  c = begin;
  p = begin;
  eof = pe;

  %% write init;
  %% write exec;

  if (p > c) {
     g_byte_array_append (data, (const guint8 *)c, p - c);
  }
}


Ragel chunk parser: compilation err, unused var

Closes #1266

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2020 Cloudius Systems, Ltd.
 */

#pragma once

#include <seastar/core/ragel.hh>
#include <seastar/core/sstring.hh>
#include <unordered_map>

namespace seastar {

%% machine chunk_head;

%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action store_name {
    _name = str();
}

action store_value {
    _value = str();
}

action store_size {
    _size = str();
}

action assign_value {
    _extensions[_name] = std::move(_value);
}

action done {
    done = true;
    fbreak;
}

crlf  = '\r\n';
sp    = ' ';
ht    = '\t';
sp_ht = sp | ht;
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';

# names correspond to the ones in RFC 7230 (with hyphens instead of underscores)
obs_text = 0x80..0xFF;
qdtext = any - (cntrl | '"' | '\\');
quoted_pair = ('\\' >{ g.mark_end(p); }) ((any - cntrl) >mark) ;
token = tchar+;
quoted_string = '"' ((qdtext | quoted_pair)* >mark %store_value) '"';
chunk_ext_name = token >mark %store_name;
chunk_ext_val = (token >mark %store_value) | quoted_string ;
chunk_size = xdigit+ >mark %store_size;
chunk_ext = (';' chunk_ext_name ('=' chunk_ext_val)? %assign_value)*;

main := chunk_size chunk_ext crlf @done;

}%%

class http_chunk_size_and_ext_parser : public ragel_parser_base<http_chunk_size_and_ext_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unordered_map<sstring, sstring> _extensions;
    sstring _name;
    sstring _value;
    sstring _size;
    state _state;
public:
    void init() {
        init_base();
        _extensions = std::unordered_map<sstring, sstring>();
        _value = "";
        _size = "";
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        [[maybe_unused]] auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            if (p == eof) {
                _state = state::eof;
            } else if (p != pe) {
                _state = state::error;
            } else {
                p = nullptr;
            }
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_extensions() {
        return std::move(_extensions);
    }
    auto get_size() {
        return std::move(_size);
    }
    bool eof() const {
        return _state == state::eof;
    }
    bool failed() const {
        return _state == state::error;
    }
};

%% machine chunk_tail;
// the headers in the chunk trailer are parsed the same way as in the request_parser.rl
%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action move {
    g.mark_end(p);
    g.mark_start(p);
}

action store_field_name {
    _field_name = str();
}

action no_mark_store_value {
    _value = get_str();
    g.mark_start(nullptr);
}

action checkpoint {
    g.mark_end(p);
    g.mark_start(p);
}

action assign_field {
    if (_headers.count(_field_name)) {
        _headers[_field_name] += sstring(",") + std::move(_value);
    } else {
        _headers[_field_name] = std::move(_value);
    }
}

action extend_field  {
    _headers[_field_name] += sstring(" ") + std::move(_value);
}

action done {
    done = true;
    fbreak;
}

crlf = '\r\n';
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';
sp = ' ';
ht = '\t';
sp_ht = sp | ht;
ows = sp_ht*;
rws = sp_ht+;

obs_text = 0x80..0xFF; # defined in RFC 7230, Section 3.2.6.
field_vchars = (graph | obs_text)+ %checkpoint;
field_content = (field_vchars ows)*;

field = tchar+ >mark %store_field_name;
value = field_content >mark %no_mark_store_value;
header_1st = field ':' ows value crlf %assign_field;
header_cont = rws value crlf %extend_field;
header = header_1st header_cont*;
main := header* crlf @done;

}%%

class http_chunk_trailer_parser : public ragel_parser_base<http_chunk_trailer_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unordered_map<sstring, sstring> _headers;
    sstring _field_name;
    sstring _value;
    state _state;
public:
    void init() {
        init_base();
        _headers = std::unordered_map<sstring, sstring>();
        _field_name = "";
        _value = "";
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        [[maybe_unused]] auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            if (p == eof) {
                _state = state::eof;
            } else if (p != pe) {
                _state = state::error;
            } else {
                p = nullptr;
            }
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_headers() {
        return std::move(_headers);
    }
    bool eof() const {
        return _state == state::eof;
    }
    bool failed() const {
        return _state == state::error;
    }
};

}

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2020 Cloudius Systems, Ltd.
 */

#pragma once

#include <seastar/core/ragel.hh>
#include <seastar/core/sstring.hh>
#include <unordered_map>

namespace seastar {

%% machine chunk_head;

%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action store_name {
    _name = str();
}

action store_value {
    _value = str();
}

action store_size {
    _size = str();
}

action assign_value {
    _extensions[_name] = std::move(_value);
}

action done {
    done = true;
    fbreak;
}

crlf  = '\r\n';
sp    = ' ';
ht    = '\t';
sp_ht = sp | ht;
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';

# names correspond to the ones in RFC 7230 (with hyphens instead of underscores)
obs_text = 0x80..0xFF;
qdtext = any - (cntrl | '"' | '\\');
quoted_pair = ('\\' >{ g.mark_end(p); }) ((any - cntrl) >mark) ;
token = tchar+;
quoted_string = '"' ((qdtext | quoted_pair)* >mark %store_value) '"';
chunk_ext_name = token >mark %store_name;
chunk_ext_val = (token >mark %store_value) | quoted_string ;
chunk_size = xdigit+ >mark %store_size;
chunk_ext = (';' chunk_ext_name ('=' chunk_ext_val)? %assign_value)*;

main := chunk_size chunk_ext crlf @done;

}%%

class http_chunk_size_and_ext_parser : public ragel_parser_base<http_chunk_size_and_ext_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unordered_map<sstring, sstring> _extensions;
    sstring _name;
    sstring _value;
    sstring _size;
    state _state;
public:
    void init() {
        init_base();
        _extensions = std::unordered_map<sstring, sstring>();
        _value = "";
        _size = "";
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            if (p == eof) {
                _state = state::eof;
            } else if (p != pe) {
                _state = state::error;
            } else {
                p = nullptr;
            }
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_extensions() {
        return std::move(_extensions);
    }
    auto get_size() {
        return std::move(_size);
    }
    bool eof() const {
        return _state == state::eof;
    }
    bool failed() const {
        return _state == state::error;
    }
};

%% machine chunk_tail;
// the headers in the chunk trailer are parsed the same way as in the request_parser.rl
%%{

access _fsm_;

action mark {
    g.mark_start(p);
}

action move {
    g.mark_end(p);
    g.mark_start(p);
}

action store_field_name {
    _field_name = str();
}

action no_mark_store_value {
    _value = get_str();
    g.mark_start(nullptr);
}

action checkpoint {
    g.mark_end(p);
    g.mark_start(p);
}

action assign_field {
    if (_headers.count(_field_name)) {
        _headers[_field_name] += sstring(",") + std::move(_value);
    } else {
        _headers[_field_name] = std::move(_value);
    }
}

action extend_field  {
    _headers[_field_name] += sstring(" ") + std::move(_value);
}

action done {
    done = true;
    fbreak;
}

crlf = '\r\n';
tchar = alpha | digit | '-' | '!' | '#' | '$' | '%' | '&' | '\'' | '*'
        | '+' | '.' | '^' | '_' | '`' | '|' | '~';
sp = ' ';
ht = '\t';
sp_ht = sp | ht;
ows = sp_ht*;
rws = sp_ht+;

obs_text = 0x80..0xFF; # defined in RFC 7230, Section 3.2.6.
field_vchars = (graph | obs_text)+ %checkpoint;
field_content = (field_vchars ows)*;

field = tchar+ >mark %store_field_name;
value = field_content >mark %no_mark_store_value;
header_1st = field ':' ows value crlf %assign_field;
header_cont = rws value crlf %extend_field;
header = header_1st header_cont*;
main := header* crlf @done;

}%%

class http_chunk_trailer_parser : public ragel_parser_base<http_chunk_trailer_parser> {
    %% write data nofinal noprefix;
public:
    enum class state {
        error,
        eof,
        done,
    };
    std::unordered_map<sstring, sstring> _headers;
    sstring _field_name;
    sstring _value;
    state _state;
public:
    void init() {
        init_base();
        _headers = std::unordered_map<sstring, sstring>();
        _field_name = "";
        _value = "";
        _state = state::eof;
        %% write init;
    }
    char* parse(char* p, char* pe, char* eof) {
        sstring_builder::guard g(_builder, p, pe);
        auto str = [this, &g, &p] { g.mark_end(p); return get_str(); };
        bool done = false;
        if (p != pe) {
            _state = state::error;
        }
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmisleading-indentation"
#endif
        %% write exec;
#ifdef __clang__
#pragma clang diagnostic pop
#endif
        if (!done) {
            if (p == eof) {
                _state = state::eof;
            } else if (p != pe) {
                _state = state::error;
            } else {
                p = nullptr;
            }
        } else {
            _state = state::done;
        }
        return p;
    }
    auto get_parsed_headers() {
        return std::move(_headers);
    }
    bool eof() const {
        return _state == state::eof;
    }
    bool failed() const {
        return _state == state::error;
    }
};

}


wrap to class

wrap to class

%%{
  machine params_tree;
  action H { @head = p }
  action T { @tail = p }
  action inject_key { inject_key }
  action push { push }
  action pop { pop }

  DELIMITER = ','                                   >H@T @{ log(:delimiter) };
  LEVEL = '('+                                      >H@T @{ log(:level) };
  END_LEVEL = ')'+                                  >H@T @{ log(:end_level); };
  INPUT = (any - LEVEL - END_LEVEL - DELIMITER)+    >H@T @{ log(:input); memo_key; };

  main := (
    INPUT
    (
      DELIMITER @inject_key @{ fnext main; } |
      END_LEVEL DELIMITER? @inject_key @pop @{ fret; } |
      LEVEL @inject_key @push @{ fcall main; }
    )
  )*;
}%%

module ParamsTree
  class Parser
    attr_accessor :data

    def initialize
      %% write data;
    end

    def parse(input)
      @data = input.unpack('c*')
      stack = []

      @log = []
      @hash = {}
      @hash_stack = []
      @key = nil

      %% write init;
      %% write exec;

      inject_key # FIXME

      #puts key
      #puts @hash.inspect
      #puts @hash_stack.inspect

      @hash_stack.first || @hash
    end

    def chars
      @data[@head..@tail]
    end

    def strokes
      chars.pack('c*')
    end

    def log(token)
      @log << { token => strokes }
    end

    def memo_key
      @key = chars
    end

    def key
      @key.pack('c*')
    end

    def inject_key
      @hash[key] = {}
    end

    def push
      @hash_stack.push @hash; @hash = @hash[key]
    end

    def pop
      @hash = @hash_stack.pop
    end

    def self.parse(string)
      self.new.parse(string)
    end
  end
end

#s = "default(all,usadasdid(zczxc(pio(xqwe)),cvb)),isdsdd(wow,er),sed(rew,tre,yrt(dfg,gfd))"
#s = "default,all"
s = "default(id,op),all"
puts s
puts ParamsTree::Parser.parse s

%%{
  machine params_parser;
  action H { @head = p }
  action T { @tail = p }
  action inject_key { inject_key }
  action push { push }
  action pop { pop }

  DELIMITER = ','                                   >H@T @{ log(:delimiter) };
  LEVEL = '('+                                      >H@T @{ log(:level) };
  END_LEVEL = ')'+                                  >H@T @{ log(:end_level); };
  INPUT = (any - LEVEL - END_LEVEL - DELIMITER)+    >H@T @{ log(:input); memo_key; };

  main := (
    INPUT
    (
      DELIMITER @inject_key @{ fnext main; } |
      END_LEVEL DELIMITER? @inject_key @pop @{ fret; } |
      LEVEL @inject_key @push @{ fcall main; }
    )
  )*;
}%%

module ParamsTree
  class Parser
    attr_accessor :data

    def initialize
      %% write data;
    end

    def process(input)
      @data = input.unpack('c*')
      stack = []

      @log = []
      @hash = {}
      @hash_stack = []
      @key = nil

      %% write init;
      %% write exec;

      puts key
      puts @hash.inspect
      puts @hash_stack.inspect

      @hash_stack.first
    end

    def chars
      @data[@head..@tail]
    end

    def strokes
      chars.pack('c*')
    end

    def log(token)
      @log << { token => strokes }
    end

    def memo_key
      @key = chars
    end

    def key
      @key.pack('c*')
    end

    def inject_key
      @hash[key] = {}
    end

    def push
      @hash_stack.push @hash; @hash = @hash[key]
    end

    def pop
      @hash = @hash_stack.pop
    end
  end
end

#s = "default(all,usadasdid(zczxc(pio(xqwe)),cvb)),isdsdd(wow,er),sed(rew,tre,yrt(dfg,gfd))"
s = "default,all"
puts s
puts ParamsTree::Parser.new.process s


identifier; on error only create AST if necessary

The finish_ident leaving action may create the AST. If an error action
follows then we need to disregard setting of the AST again.

closes #14

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_ident {
    @buffers[:ident] = []
  }

  action append_ident {
    (@buffers[:ident] ||= []) << fc
  }

  action finish_ident {
    @buffers[:ident] = s(:identifier,
                         utf8_string(@buffers[:ident]))
  }

  action yield_complete_ident {
    yield @buffers[:ident]
  }

  action error_ident {
    unless @buffers[:ident].is_a?(::AST::Node)
      @buffers[:ident] ||= []
      @buffers[:ident]   = s(:identifier,
                             utf8_string(@buffers[:ident]).sub(/\n$/, ''))
    end
  }

  action yield_error_ident {
    @buffers[:ident] ||= []
    yield @buffers[:ident]
  }

  IDENT = [a-zA-Z0-9_]+ >start_ident $append_ident %finish_ident $err(error_ident);

  ident := IDENT $err(yield_error_ident) %yield_complete_ident NL;
}%%
=end
# end: ragel

require          'ast'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BEL
  module Parsers
    module Common
      module Identifier

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include AST::Sexp
          include BEL::Parser::Buffer

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers = {}
            data = @content.unpack('C*')
            p   = 0
            pe  = data.length
            eof = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BEL::Parsers::Common::Identifier.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_ident {
    @buffers[:ident] = []
  }

  action append_ident {
    (@buffers[:ident] ||= []) << fc
  }

  action finish_ident {
    @buffers[:ident] = s(:identifier,
                         utf8_string(@buffers[:ident]))
  }

  action yield_complete_ident {
    yield @buffers[:ident]
  }

  action error_ident {
    @buffers[:ident] ||= []
    @buffers[:ident]   = s(:identifier,
                           utf8_string(@buffers[:ident]).sub(/\n$/, ''))
  }

  action yield_error_ident {
    @buffers[:ident] ||= []
    yield @buffers[:ident]
  }

  IDENT = [a-zA-Z0-9_]+ >start_ident $append_ident %finish_ident $err(error_ident);

  ident := IDENT $err(yield_error_ident) %yield_complete_ident NL;
}%%
=end
# end: ragel

require          'ast'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BEL
  module Parsers
    module Common
      module Identifier

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include AST::Sexp
          include BEL::Parser::Buffer

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers = {}
            data = @content.unpack('C*')
            p   = 0
            pe  = data.length
            eof = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BEL::Parsers::Common::Identifier.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Constant should support namescope.

Constant should support namescope.

Sourcify.require_rb('proc', 'scanner', 'extensions')

module Sourcify
  module Proc
    module Scanner #:nodoc:all

%%{
  machine proc_scanner;

  kw_class  = 'class';
  kw_module = 'module';
  kw_def    = 'def';
  kw_begin  = 'begin';
  kw_case   = 'case';
  kw_if     = 'if';
  kw_unless = 'unless';
  kw_do     = 'do';
  kw_then   = 'then';
  kw_for    = 'for';
  kw_while  = 'while';
  kw_until  = 'until';
  kw_end    = 'end';

  const   = upper . (alnum | '_')* . ('::' . upper . (alnum | '_')*)*;
  var     = (lower | '_') . (alnum | '_')*;
  symbol  = ':' . (var | const);
  assoc   = '=>';
  label   = (var | const) . ':';

  lparen  = '(';
  rparen  = ')';
  lbrace  = '{';
  rbrace  = '}';
  smcolon = ';';
  newline = '\n';
  ospaces = space* -- newline;
  mspaces = space+ -- newline;

  ## MACHINE >> New statement
  new_statement := |*

    kw_for | kw_while | kw_until => {
      push(:kw_do_alias2, ts, te)
      increment_counter(:do_end, 0..1)
      fgoto main;
    };

    (
      kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless |
      kw_class . ospaces . '<<' . ospaces . ^newline+
    ) => {
      push(:kw_do_alias1, ts, te)
      increment_counter(:do_end, 1)
      fgoto main;
    };

    ospaces => { push(:space, ts, te) };
    any => { fhold; fgoto main; };

  *|;

  ## MACHINE >> One-liner comment
  per_line_comment := |*
    ^newline* => {
      push(:comment, ts.pred, te)
      fgoto main;
    };
  *|;

  ## MACHINE >> Block comment
  block_comment := |*
    any* . newline . '=end' . ospaces . ^newline* => {
      unless push_comment(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Heredoc
  heredoc := |*
    ^newline* . newline . ospaces . ^newline+ => {
      unless push_heredoc(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Double quote string
  double_quote_str := |*
    delimiter = [\~\`\!\@\#\$\%\^\&\*\)\-\_\+\=\}\]\:\;\'\"\>\,\.\?\/\|\\];
    (^delimiter* . delimiter) -- ('\\' . delimiter) => {
      unless push_dstring(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Main
  main := |*

    ## == Start/end of do..end block

    kw_do => {
      push(:kw_do, ts, te)
      increment_counter(:do_end)
      fgoto new_statement;
    };

    kw_end => {
      push(:kw_end, ts, te)
      decrement_counter(:do_end)
    };

    ## == Start/end of {...} block

    lbrace => {
      push(:lbrace, ts, te)
      increment_counter(:brace)
    };

    rbrace => {
      push(:rbrace, ts, te)
      decrement_counter(:brace)
    };

    assoc => {
      push(:assoc, ts, te)
      fix_counter_false_start(:brace)
    };

    label => {
      push_label(ts, te)
      fix_counter_false_start(:brace)
    };

    ## == New statement

    newline => {
      push(:newline, ts, te)
      increment_lineno
      fgoto new_statement;
    };

    smcolon => {
      push(:smcolon, ts, te)
      fgoto new_statement;
    };

    lparen => {
      push(:lparen, ts, te)
      fgoto new_statement;
    };

    kw_then => {
      push(:kw_then, ts, te)
      fgoto new_statement;
    };

    ## == Comment

    '#' => {
      fgoto per_line_comment;
    };

    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
      push_comment(ts, te)
      increment_lineno
      fgoto block_comment;
    };

    ## == Misc

    var     => { push(:var, ts, te) };
    const   => { push(:const, ts, te) };
    symbol  => { push(:symbol, ts, te) };
    mspaces => { push(:space, ts, te) };
    any     => { push(:any, ts, te) };

    ## == Heredoc

    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
      push_heredoc(ts, te)
      increment_lineno
      fgoto heredoc;
    };

    ## == Single quote strings

    sqs1  = "'" . ([^\']* | ([^\'\\]*[\\][\'][^\']*)*) . '\\\\'* . "'";
    sqs2  = '~' . ([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) . '\\\\'* . '~';
    sqs3  = '`' . ([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) . '\\\\'* . '`';
    sqs4  = '!' . ([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) . '\\\\'* . '!';
    sqs5  = '@' . ([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) . '\\\\'* . '@';
    sqs6  = '#' . ([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) . '\\\\'* . '#';
    sqs7  = '$' . ([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) . '\\\\'* . '$';
    sqs8  = '%' . ([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) . '\\\\'* . '%';
    sqs9  = '^' . ([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) . '\\\\'* . '^';
    sqs10 = '&' . ([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) . '\\\\'* . '&';
    sqs11 = '*' . ([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) . '\\\\'* . '*';
    sqs12 = '-' . ([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) . '\\\\'* . '-';
    sqs13 = '_' . ([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) . '\\\\'* . '_';
    sqs14 = '+' . ([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) . '\\\\'* . '+';
    sqs15 = '=' . ([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) . '\\\\'* . '=';
    sqs16 = '<' . ([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) . '\\\\'* . '>';
    sqs17 = '|' . ([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) . '\\\\'* . '|';
    sqs18 = ':' . ([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) . '\\\\'* . ':';
    sqs19 = ';' . ([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) . '\\\\'* . ';';
    sqs20 = '"' . ([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) . '\\\\'* . '"';
    sqs21 = ',' . ([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) . '\\\\'* . ',';
    sqs22 = '.' . ([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) . '\\\\'* . '.';
    sqs23 = '?' . ([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) . '\\\\'* . '?';
    sqs24 = '/' . ([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) . '\\\\'* . '/';
    sqs25 = '{' . ([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) . '\\\\'* . '}';
    sqs26 = '[' . ([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) . '\\\\'* . ']';
    sqs27 = '(' . ([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) . '\\\\'* . ')';
    sqs28 = '\\' . ([^\\]* | ([^\)]*[\\][\\][^\\]*)*) . '\\';

    sqm = ('%q' | '%w'); (
      sqs1        | sqm . sqs1  | sqm . sqs2  | sqm . sqs3  | sqm . sqs4  |
      sqm . sqs5  | sqm . sqs6  | sqm . sqs7  | sqm . sqs8  | sqm . sqs9  |
      sqm . sqs10 | sqm . sqs11 | sqm . sqs12 | sqm . sqs13 | sqm . sqs14 |
      sqm . sqs15 | sqm . sqs16 | sqm . sqs17 | sqm . sqs18 | sqm . sqs19 |
      sqm . sqs20 | sqm . sqs21 | sqm . sqs22 | sqm . sqs23 | sqm . sqs24 |
      sqm . sqs25 | sqm . sqs26 | sqm . sqs27 | sqm . sqs28
    ) => {
      push(:sstring, ts, te)
    };

    ## == Double quote strings

    dqs1  = '"' . (([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) -- '#{') . '\\\\'* . ('"' | '#{');
    dqs2  = '`' . (([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) -- '#{') . '\\\\'* . ('`' | '#{');
    dqs3  = '/' . (([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) -- '#{') . '\\\\'* . ('/' | '#{');
    dqs4  = "'" . (([^\']* | ([^\'\\]*[\\][\'][^\']*)*) -- '#{') . '\\\\'* . ("'" | '#{');
    dqs5  = '~' . (([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) -- '#{') . '\\\\'* . ('~' | '#{');
    dqs6  = '!' . (([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) -- '#{') . '\\\\'* . ('!' | '#{');
    dqs7  = '@' . (([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) -- '#{') . '\\\\'* . ('@' | '#{');
    dqs8  = '#' . (([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) -- '#{') . '\\\\'* . ('#' | '#{');
    dqs9  = '$' . (([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) -- '#{') . '\\\\'* . ('$' | '#{');
    dqs10 = '%' . (([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) -- '#{') . '\\\\'* . ('%' | '#{');
    dqs11 = '^' . (([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) -- '#{') . '\\\\'* . ('^' | '#{');
    dqs12 = '&' . (([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) -- '#{') . '\\\\'* . ('&' | '#{');
    dqs13 = '*' . (([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) -- '#{') . '\\\\'* . ('*' | '#{');
    dqs14 = '-' . (([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) -- '#{') . '\\\\'* . ('-' | '#{');
    dqs15 = '_' . (([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) -- '#{') . '\\\\'* . ('_' | '#{');
    dqs16 = '+' . (([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) -- '#{') . '\\\\'* . ('+' | '#{');
    dqs17 = '=' . (([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) -- '#{') . '\\\\'* . ('=' | '#{');
    dqs18 = '<' . (([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) -- '#{') . '\\\\'* . ('>' | '#{');
    dqs19 = '|' . (([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) -- '#{') . '\\\\'* . ('|' | '#{');
    dqs20 = ':' . (([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) -- '#{') . '\\\\'* . (':' | '#{');
    dqs21 = ';' . (([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) -- '#{') . '\\\\'* . (';' | '#{');
    dqs22 = ',' . (([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) -- '#{') . '\\\\'* . (',' | '#{');
    dqs23 = '.' . (([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) -- '#{') . '\\\\'* . ('.' | '#{');
    dqs24 = '?' . (([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) -- '#{') . '\\\\'* . ('?' | '#{');
    dqs25 = '{' . (([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) -- '#{') . '\\\\'* . ('}' | '#{');
    dqs26 = '[' . (([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) -- '#{') . '\\\\'* . (']' | '#{');
    dqs27 = '(' . (([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) -- '#{') . '\\\\'* . (')' | '#{');
    dqs28 = '\\' . (([^\\]* | ([^\\]*[\\][\\][^\\]*)*) -- '#{') . ('\\' | '#{');

    dqm = ('%Q' | '%W' | '%x' | '%r' | '%'); (
      dqs1        | dqs2        | dqs3        | dqm . dqs1  | dqm . dqs2  |
      dqm . dqs3  | dqm . dqs4  | dqm . dqs5  | dqm . dqs6  | dqm . dqs7  |
      dqm . dqs8  | dqm . dqs9  | dqm . dqs10 | dqm . dqs11 | dqm . dqs12 |
      dqm . dqs13 | dqm . dqs14 | dqm . dqs15 | dqm . dqs16 | dqm . dqs17 |
      dqm . dqs18 | dqm . dqs19 | dqm . dqs20 | dqm . dqs21 | dqm . dqs22 |
      dqm . dqs23 | dqm . dqs24 | dqm . dqs25 | dqm . dqs26 | dqm . dqs27 |
      dqm . dqs28
    ) => {
      if push_dstring(ts, te)
        fgoto double_quote_str;
      end
    };

  *|;

}%%
%% write data;

      extend Scanner::Extensions

      def self.execute!
        data = @data
        eof = data.length
        %% write init;
        %% write exec;
      end

    end
  end
end

Sourcify.require_rb('proc', 'scanner', 'extensions')

module Sourcify
  module Proc
    module Scanner #:nodoc:all

%%{
  machine proc_scanner;

  kw_class  = 'class';
  kw_module = 'module';
  kw_def    = 'def';
  kw_begin  = 'begin';
  kw_case   = 'case';
  kw_if     = 'if';
  kw_unless = 'unless';
  kw_do     = 'do';
  kw_then   = 'then';
  kw_for    = 'for';
  kw_while  = 'while';
  kw_until  = 'until';
  kw_end    = 'end';

  const   = upper . (alnum | '_')*;
  var     = (lower | '_') . (alnum | '_')*;
  symbol  = ':' . (var | const);
  assoc   = '=>';
  label   = (var | const) . ':';

  lparen  = '(';
  rparen  = ')';
  lbrace  = '{';
  rbrace  = '}';
  smcolon = ';';
  newline = '\n';
  ospaces = space* -- newline;
  mspaces = space+ -- newline;

  ## MACHINE >> New statement
  new_statement := |*

    kw_for | kw_while | kw_until => {
      push(:kw_do_alias2, ts, te)
      increment_counter(:do_end, 0..1)
      fgoto main;
    };

    (
      kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless |
      kw_class . ospaces . '<<' . ospaces . ^newline+
    ) => {
      push(:kw_do_alias1, ts, te)
      increment_counter(:do_end, 1)
      fgoto main;
    };

    ospaces => { push(:space, ts, te) };
    any => { fhold; fgoto main; };

  *|;

  ## MACHINE >> One-liner comment
  per_line_comment := |*
    ^newline* => {
      push(:comment, ts.pred, te)
      fgoto main;
    };
  *|;

  ## MACHINE >> Block comment
  block_comment := |*
    any* . newline . '=end' . ospaces . ^newline* => {
      unless push_comment(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Heredoc
  heredoc := |*
    ^newline* . newline . ospaces . ^newline+ => {
      unless push_heredoc(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Double quote string
  double_quote_str := |*
    delimiter = [\~\`\!\@\#\$\%\^\&\*\)\-\_\+\=\}\]\:\;\'\"\>\,\.\?\/\|\\];
    (^delimiter* . delimiter) -- ('\\' . delimiter) => {
      unless push_dstring(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Main
  main := |*

    ## == Start/end of do..end block

    kw_do => {
      push(:kw_do, ts, te)
      increment_counter(:do_end)
      fgoto new_statement;
    };

    kw_end => {
      push(:kw_end, ts, te)
      decrement_counter(:do_end)
    };

    ## == Start/end of {...} block

    lbrace => {
      push(:lbrace, ts, te)
      increment_counter(:brace)
    };

    rbrace => {
      push(:rbrace, ts, te)
      decrement_counter(:brace)
    };

    assoc => {
      push(:assoc, ts, te)
      fix_counter_false_start(:brace)
    };

    label => {
      push_label(ts, te)
      fix_counter_false_start(:brace)
    };

    ## == New statement

    newline => {
      push(:newline, ts, te)
      increment_lineno
      fgoto new_statement;
    };

    smcolon => {
      push(:smcolon, ts, te)
      fgoto new_statement;
    };

    lparen => {
      push(:lparen, ts, te)
      fgoto new_statement;
    };

    kw_then => {
      push(:kw_then, ts, te)
      fgoto new_statement;
    };

    ## == Comment

    '#' => {
      fgoto per_line_comment;
    };

    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
      push_comment(ts, te)
      increment_lineno
      fgoto block_comment;
    };

    ## == Misc

    var     => { push(:var, ts, te) };
    const   => { push(:const, ts, te) };
    symbol  => { push(:symbol, ts, te) };
    mspaces => { push(:space, ts, te) };
    any     => { push(:any, ts, te) };

    ## == Heredoc

    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
      push_heredoc(ts, te)
      increment_lineno
      fgoto heredoc;
    };

    ## == Single quote strings

    sqs1  = "'" . ([^\']* | ([^\'\\]*[\\][\'][^\']*)*) . '\\\\'* . "'";
    sqs2  = '~' . ([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) . '\\\\'* . '~';
    sqs3  = '`' . ([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) . '\\\\'* . '`';
    sqs4  = '!' . ([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) . '\\\\'* . '!';
    sqs5  = '@' . ([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) . '\\\\'* . '@';
    sqs6  = '#' . ([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) . '\\\\'* . '#';
    sqs7  = '$' . ([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) . '\\\\'* . '$';
    sqs8  = '%' . ([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) . '\\\\'* . '%';
    sqs9  = '^' . ([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) . '\\\\'* . '^';
    sqs10 = '&' . ([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) . '\\\\'* . '&';
    sqs11 = '*' . ([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) . '\\\\'* . '*';
    sqs12 = '-' . ([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) . '\\\\'* . '-';
    sqs13 = '_' . ([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) . '\\\\'* . '_';
    sqs14 = '+' . ([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) . '\\\\'* . '+';
    sqs15 = '=' . ([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) . '\\\\'* . '=';
    sqs16 = '<' . ([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) . '\\\\'* . '>';
    sqs17 = '|' . ([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) . '\\\\'* . '|';
    sqs18 = ':' . ([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) . '\\\\'* . ':';
    sqs19 = ';' . ([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) . '\\\\'* . ';';
    sqs20 = '"' . ([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) . '\\\\'* . '"';
    sqs21 = ',' . ([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) . '\\\\'* . ',';
    sqs22 = '.' . ([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) . '\\\\'* . '.';
    sqs23 = '?' . ([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) . '\\\\'* . '?';
    sqs24 = '/' . ([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) . '\\\\'* . '/';
    sqs25 = '{' . ([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) . '\\\\'* . '}';
    sqs26 = '[' . ([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) . '\\\\'* . ']';
    sqs27 = '(' . ([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) . '\\\\'* . ')';
    sqs28 = '\\' . ([^\\]* | ([^\)]*[\\][\\][^\\]*)*) . '\\';

    sqm = ('%q' | '%w'); (
      sqs1        | sqm . sqs1  | sqm . sqs2  | sqm . sqs3  | sqm . sqs4  |
      sqm . sqs5  | sqm . sqs6  | sqm . sqs7  | sqm . sqs8  | sqm . sqs9  |
      sqm . sqs10 | sqm . sqs11 | sqm . sqs12 | sqm . sqs13 | sqm . sqs14 |
      sqm . sqs15 | sqm . sqs16 | sqm . sqs17 | sqm . sqs18 | sqm . sqs19 |
      sqm . sqs20 | sqm . sqs21 | sqm . sqs22 | sqm . sqs23 | sqm . sqs24 |
      sqm . sqs25 | sqm . sqs26 | sqm . sqs27 | sqm . sqs28
    ) => {
      push(:sstring, ts, te)
    };

    ## == Double quote strings

    dqs1  = '"' . (([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) -- '#{') . '\\\\'* . ('"' | '#{');
    dqs2  = '`' . (([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) -- '#{') . '\\\\'* . ('`' | '#{');
    dqs3  = '/' . (([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) -- '#{') . '\\\\'* . ('/' | '#{');
    dqs4  = "'" . (([^\']* | ([^\'\\]*[\\][\'][^\']*)*) -- '#{') . '\\\\'* . ("'" | '#{');
    dqs5  = '~' . (([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) -- '#{') . '\\\\'* . ('~' | '#{');
    dqs6  = '!' . (([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) -- '#{') . '\\\\'* . ('!' | '#{');
    dqs7  = '@' . (([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) -- '#{') . '\\\\'* . ('@' | '#{');
    dqs8  = '#' . (([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) -- '#{') . '\\\\'* . ('#' | '#{');
    dqs9  = '$' . (([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) -- '#{') . '\\\\'* . ('$' | '#{');
    dqs10 = '%' . (([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) -- '#{') . '\\\\'* . ('%' | '#{');
    dqs11 = '^' . (([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) -- '#{') . '\\\\'* . ('^' | '#{');
    dqs12 = '&' . (([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) -- '#{') . '\\\\'* . ('&' | '#{');
    dqs13 = '*' . (([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) -- '#{') . '\\\\'* . ('*' | '#{');
    dqs14 = '-' . (([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) -- '#{') . '\\\\'* . ('-' | '#{');
    dqs15 = '_' . (([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) -- '#{') . '\\\\'* . ('_' | '#{');
    dqs16 = '+' . (([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) -- '#{') . '\\\\'* . ('+' | '#{');
    dqs17 = '=' . (([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) -- '#{') . '\\\\'* . ('=' | '#{');
    dqs18 = '<' . (([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) -- '#{') . '\\\\'* . ('>' | '#{');
    dqs19 = '|' . (([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) -- '#{') . '\\\\'* . ('|' | '#{');
    dqs20 = ':' . (([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) -- '#{') . '\\\\'* . (':' | '#{');
    dqs21 = ';' . (([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) -- '#{') . '\\\\'* . (';' | '#{');
    dqs22 = ',' . (([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) -- '#{') . '\\\\'* . (',' | '#{');
    dqs23 = '.' . (([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) -- '#{') . '\\\\'* . ('.' | '#{');
    dqs24 = '?' . (([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) -- '#{') . '\\\\'* . ('?' | '#{');
    dqs25 = '{' . (([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) -- '#{') . '\\\\'* . ('}' | '#{');
    dqs26 = '[' . (([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) -- '#{') . '\\\\'* . (']' | '#{');
    dqs27 = '(' . (([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) -- '#{') . '\\\\'* . (')' | '#{');
    dqs28 = '\\' . (([^\\]* | ([^\\]*[\\][\\][^\\]*)*) -- '#{') . ('\\' | '#{');

    dqm = ('%Q' | '%W' | '%x' | '%r' | '%'); (
      dqs1        | dqs2        | dqs3        | dqm . dqs1  | dqm . dqs2  |
      dqm . dqs3  | dqm . dqs4  | dqm . dqs5  | dqm . dqs6  | dqm . dqs7  |
      dqm . dqs8  | dqm . dqs9  | dqm . dqs10 | dqm . dqs11 | dqm . dqs12 |
      dqm . dqs13 | dqm . dqs14 | dqm . dqs15 | dqm . dqs16 | dqm . dqs17 |
      dqm . dqs18 | dqm . dqs19 | dqm . dqs20 | dqm . dqs21 | dqm . dqs22 |
      dqm . dqs23 | dqm . dqs24 | dqm . dqs25 | dqm . dqs26 | dqm . dqs27 |
      dqm . dqs28
    ) => {
      if push_dstring(ts, te)
        fgoto double_quote_str;
      end
    };

  *|;

}%%
%% write data;

      extend Scanner::Extensions

      def self.execute!
        data = @data
        eof = data.length
        %% write init;
        %% write exec;
      end

    end
  end
end


Fix warning

Fix warning

#include <stdlib.h>
#include "readstat_sav.h"
#include "readstat_sav_parse.h"

typedef struct varlookup {
    char      name[8*4+1];
    int       index;
} varlookup_t;

static int compare_key_varlookup(const void *elem1, const void *elem2) {
    const char *key = (const char *)elem1;
    const varlookup_t *v = (const varlookup_t *)elem2;
    return strcmp(key, v->name);
}

static int compare_varlookups(const void *elem1, const void *elem2) {
    const varlookup_t *v1 = (const varlookup_t *)elem1;
    const varlookup_t *v2 = (const varlookup_t *)elem2;
    return strcmp(v1->name, v2->name);
}

static void build_lookup_table(varlookup_t *table, int var_count, sav_ctx_t *ctx) {
    int offset = 0;
    int i;
    for (i=0; i<ctx->var_index;) {
        spss_varinfo_t *info = &ctx->varinfo[i];
        varlookup_t *entry = &table[offset++];

        memcpy(entry->name, info->name, sizeof(info->name));
        entry->index = info->index;

        i += info->n_segments;
    }
    qsort(table, var_count, sizeof(varlookup_t), &compare_varlookups);
}

%%{
    machine sav_long_variable_parse;
    write data nofinal noerror;
    alphtype unsigned char;
}%%

readstat_error_t sav_parse_long_variable_names_record(void *data, int count, sav_ctx_t *ctx) {
    unsigned char *c_data = (unsigned char *)data;
    int var_count = ctx->var_index;
    varlookup_t *table = malloc(var_count * sizeof(varlookup_t));
    readstat_error_t retval = READSTAT_OK;

    char temp_key[4*8+1];
    char temp_val[4*64+1];
    unsigned char *str_start = NULL;
    size_t str_len = 0;

    char error_buf[8192];
    unsigned char *p = NULL;
    unsigned char *pe = NULL;
    unsigned char *output_buffer = NULL;

    build_lookup_table(table, var_count, ctx);

    if (ctx->converter) {
        size_t input_len = count;
        size_t output_len = input_len * 4;
        pe = p = output_buffer = malloc(output_len);
        size_t status = iconv(ctx->converter,
                (readstat_iconv_inbuf_t)&data, &input_len,
                (char **)&pe, &output_len);
        if (status == (size_t)-1) {
            free(output_buffer);
            return READSTAT_ERROR_PARSE;
        }
    } else {
        p = c_data;
        pe = c_data + count;
    }
    unsigned char *eof = pe;

    int cs;

    %%{
        action set_long_name {
            varlookup_t *found = bsearch(temp_key, table, var_count, sizeof(varlookup_t), &compare_key_varlookup);
            if (found) {
                memcpy(ctx->varinfo[found->index].longname, temp_val, str_len);
                ctx->varinfo[found->index].longname[str_len] = '\0';
            } else if (ctx->error_handler) {
                snprintf(error_buf, sizeof(error_buf), "Failed to find %s\n", temp_key);
                ctx->error_handler(error_buf, ctx->user_ctx);
            }
        }

        action copy_key {
            memcpy(temp_key, str_start, str_len);
            temp_key[str_len] = '\0';
        }

        action copy_value {
            memcpy(temp_val, str_start, str_len);
            temp_val[str_len] = '\0';
        }

        non_ascii_character = ( # UTF-8 byte sequences
                0xC0..0xDF 0x80..0xBF |
                0xE0..0xEF (0x80..0xBF){2} |
                0xF0..0xF7 (0x80..0xBF){3}
                );

        key = ( ( non_ascii_character | [A-Z@] ) ( non_ascii_character | [A-Z0-9@#$_\.] ){0,7} ) >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        value = ( non_ascii_character | print ){1,64} >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        keyval = ( key %copy_key "=" value %copy_value ) %set_long_name;

        main := keyval ("\t" keyval)*  "\t"?;

        write init;
        write exec;
    }%%

    if (cs < %%{ write first_final; }%%|| p != pe) {
        if (ctx->error_handler) {
            snprintf(error_buf, sizeof(error_buf), "Error parsing string \"%s\" around byte #%ld/%d, character %c\n",
                    (char *)data, (long)(p - c_data), count, *p);
            ctx->error_handler(error_buf, ctx->user_ctx);
        }
        retval = READSTAT_ERROR_PARSE;
    }

    if (table)
        free(table);
    if (output_buffer)
        free(output_buffer);

    /* suppress warning */
    (void)sav_long_variable_parse_en_main;

    return retval;
}

%%{
    machine sav_very_long_string_parse;
    write data nofinal noerror;
    alphtype unsigned char;
}%%

readstat_error_t sav_parse_very_long_string_record(void *data, int count, sav_ctx_t *ctx) {
    unsigned char *c_data = (unsigned char *)data;
    int var_count = ctx->var_index;
    varlookup_t *table = malloc(var_count * sizeof(varlookup_t));
    readstat_error_t retval = READSTAT_OK;

    char temp_key[8*4+1];
    int temp_val = 0;
    unsigned char *str_start = NULL;
    size_t str_len = 0;

    size_t error_buf_len = 1024 + count;
    char *error_buf = malloc(error_buf_len);
    unsigned char *p = NULL;
    unsigned char *pe = NULL;

    unsigned char *output_buffer = NULL;

    build_lookup_table(table, var_count, ctx);

    if (ctx->converter) {
        size_t input_len = count;
        size_t output_len = input_len * 4;

        pe = p = output_buffer = malloc(output_len);

        size_t status = iconv(ctx->converter,
                (readstat_iconv_inbuf_t)&data, &input_len,
                (char **)&pe, &output_len);
        if (status == (size_t)-1) {
            free(output_buffer);
            return READSTAT_ERROR_PARSE;
        }
    } else {
        p = c_data;
        pe = c_data + count;
    }

    int cs;

    %%{
        action set_width {
            varlookup_t *found = bsearch(temp_key, table, var_count, sizeof(varlookup_t), &compare_key_varlookup);
            if (found) {
                ctx->varinfo[found->index].string_length = temp_val;
            }
        }

        action copy_key {
            memcpy(temp_key, str_start, str_len);
            temp_key[str_len] = '\0';
        }

        action incr_val {
            if (fc != '\0') {
                temp_val = 10 * temp_val + (fc - '0');
            }
        }

        non_ascii_character = ( # UTF-8 byte sequences
                0xC0..0xDF 0x80..0xBF |
                0xE0..0xEF (0x80..0xBF){2} |
                0xF0..0xF7 (0x80..0xBF){3}
                );

        key = ( ( non_ascii_character | [A-Z@] ) ( non_ascii_character | [A-Z0-9@#$_\.] ){0,7} ) >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        value = [0-9]+ >{ temp_val = 0; } $incr_val;

        keyval = ( key %copy_key "=" value ) %set_width;

        main := keyval ("\0"+ "\t" keyval)* "\0"+ "\t"?;

        write init;
        write exec;
    }%%

    if (cs < %%{ write first_final; }%% || p != pe) {
        if (ctx->error_handler) {
            snprintf(error_buf, error_buf_len, "Parsed %ld of %ld bytes\nRemaining bytes: %s\n", (long)(p - c_data), (long)(pe - c_data), p);
            ctx->error_handler(error_buf, ctx->user_ctx);
        }
        retval = READSTAT_ERROR_PARSE;
    }

    if (table)
        free(table);
    if (output_buffer)
        free(output_buffer);
    if (error_buf)
        free(error_buf);

    /* suppress warning */
    (void)sav_very_long_string_parse_en_main;

    return retval;
}

#include <stdlib.h>
#include "readstat_sav.h"
#include "readstat_sav_parse.h"

typedef struct varlookup {
    char      name[8*4+1];
    int       index;
} varlookup_t;

static int compare_key_varlookup(const void *elem1, const void *elem2) {
    const char *key = (const char *)elem1;
    const varlookup_t *v = (const varlookup_t *)elem2;
    return strcmp(key, v->name);
}

static int compare_varlookups(const void *elem1, const void *elem2) {
    const varlookup_t *v1 = (const varlookup_t *)elem1;
    const varlookup_t *v2 = (const varlookup_t *)elem2;
    return strcmp(v1->name, v2->name);
}

static void build_lookup_table(varlookup_t *table, int var_count, sav_ctx_t *ctx) {
    int offset = 0;
    int i;
    for (i=0; i<ctx->var_index;) {
        spss_varinfo_t *info = &ctx->varinfo[i];
        varlookup_t *entry = &table[offset++];

        memcpy(entry->name, info->name, sizeof(info->name));
        entry->index = info->index;

        i += info->n_segments;
    }
    qsort(table, var_count, sizeof(varlookup_t), &compare_varlookups);
}

%%{
    machine sav_long_variable_parse;
    write data nofinal noerror;
    alphtype unsigned char;
}%%

readstat_error_t sav_parse_long_variable_names_record(void *data, int count, sav_ctx_t *ctx) {
    unsigned char *c_data = (unsigned char *)data;
    int var_count = ctx->var_index;
    varlookup_t *table = malloc(var_count * sizeof(varlookup_t));
    readstat_error_t retval = READSTAT_OK;

    char temp_key[4*8+1];
    char temp_val[4*64+1];
    unsigned char *str_start = NULL;
    size_t str_len = 0;

    char error_buf[8192];
    unsigned char *p = NULL;
    unsigned char *pe = NULL;
    unsigned char *output_buffer = NULL;

    build_lookup_table(table, var_count, ctx);

    if (ctx->converter) {
        size_t input_len = count;
        size_t output_len = input_len * 4;
        pe = p = output_buffer = malloc(output_len);
        size_t status = iconv(ctx->converter,
                (readstat_iconv_inbuf_t)&data, &input_len,
                (char **)&pe, &output_len);
        if (status == (size_t)-1) {
            free(output_buffer);
            return READSTAT_ERROR_PARSE;
        }
    } else {
        p = c_data;
        pe = c_data + count;
    }
    unsigned char *eof = pe;

    int cs;

    %%{
        action set_long_name {
            varlookup_t *found = bsearch(temp_key, table, var_count, sizeof(varlookup_t), &compare_key_varlookup);
            if (found) {
                memcpy(ctx->varinfo[found->index].longname, temp_val, str_len);
                ctx->varinfo[found->index].longname[str_len] = '\0';
            } else if (ctx->error_handler) {
                snprintf(error_buf, sizeof(error_buf), "Failed to find %s\n", temp_key);
                ctx->error_handler(error_buf, ctx->user_ctx);
            }
        }

        action copy_key {
            memcpy(temp_key, str_start, str_len);
            temp_key[str_len] = '\0';
        }

        action copy_value {
            memcpy(temp_val, str_start, str_len);
            temp_val[str_len] = '\0';
        }

        non_ascii_character = ( # UTF-8 byte sequences
                0xC0..0xDF 0x80..0xBF |
                0xE0..0xEF (0x80..0xBF){2} |
                0xF0..0xF7 (0x80..0xBF){3}
                );

        key = ( ( non_ascii_character | [A-Z@] ) ( non_ascii_character | [A-Z0-9@#$_\.] ){0,7} ) >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        value = ( non_ascii_character | print ){1,64} >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        keyval = ( key %copy_key "=" value %copy_value ) %set_long_name;

        main := keyval ("\t" keyval)*  "\t"?;

        write init;
        write exec;
    }%%

    if (cs < %%{ write first_final; }%%|| p != pe) {
        if (ctx->error_handler) {
            snprintf(error_buf, sizeof(error_buf), "Error parsing string \"%s\" around byte #%ld/%d, character %c\n",
                    (char *)data, (long)(p - c_data), count, *p);
            ctx->error_handler(error_buf, ctx->user_ctx);
        }
        retval = READSTAT_ERROR_PARSE;
    }

    if (table)
        free(table);
    if (output_buffer)
        free(output_buffer);

    /* suppress warning */
    (void)sav_long_variable_parse_en_main;

    return retval;
}

%%{
    machine sav_very_long_string_parse;
    write data nofinal noerror;
    alphtype unsigned char;
}%%

readstat_error_t sav_parse_very_long_string_record(void *data, int count, sav_ctx_t *ctx) {
    unsigned char *c_data = (unsigned char *)data;
    int var_count = ctx->var_index;
    varlookup_t *table = malloc(var_count * sizeof(varlookup_t));
    readstat_error_t retval = READSTAT_OK;

    char temp_key[8*4+1];
    int temp_val;
    unsigned char *str_start = NULL;
    size_t str_len = 0;

    size_t error_buf_len = 1024 + count;
    char *error_buf = malloc(error_buf_len);
    unsigned char *p = NULL;
    unsigned char *pe = NULL;

    unsigned char *output_buffer = NULL;

    build_lookup_table(table, var_count, ctx);

    if (ctx->converter) {
        size_t input_len = count;
        size_t output_len = input_len * 4;

        pe = p = output_buffer = malloc(output_len);

        size_t status = iconv(ctx->converter,
                (readstat_iconv_inbuf_t)&data, &input_len,
                (char **)&pe, &output_len);
        if (status == (size_t)-1) {
            free(output_buffer);
            return READSTAT_ERROR_PARSE;
        }
    } else {
        p = c_data;
        pe = c_data + count;
    }

    int cs;

    %%{
        action set_width {
            varlookup_t *found = bsearch(temp_key, table, var_count, sizeof(varlookup_t), &compare_key_varlookup);
            if (found) {
                ctx->varinfo[found->index].string_length = temp_val;
            }
        }

        action copy_key {
            memcpy(temp_key, str_start, str_len);
            temp_key[str_len] = '\0';
        }

        action incr_val {
            if (fc != '\0') {
                temp_val = 10 * temp_val + (fc - '0');
            }
        }

        non_ascii_character = ( # UTF-8 byte sequences
                0xC0..0xDF 0x80..0xBF |
                0xE0..0xEF (0x80..0xBF){2} |
                0xF0..0xF7 (0x80..0xBF){3}
                );

        key = ( ( non_ascii_character | [A-Z@] ) ( non_ascii_character | [A-Z0-9@#$_\.] ){0,7} ) >{ str_start = fpc; } %{ str_len = fpc - str_start; };

        value = [0-9]+ >{ temp_val = 0; } $incr_val;

        keyval = ( key %copy_key "=" value ) %set_width;

        main := keyval ("\0"+ "\t" keyval)* "\0"+ "\t"?;

        write init;
        write exec;
    }%%

    if (cs < %%{ write first_final; }%% || p != pe) {
        if (ctx->error_handler) {
            snprintf(error_buf, error_buf_len, "Parsed %ld of %ld bytes\nRemaining bytes: %s\n", (long)(p - c_data), (long)(pe - c_data), p);
            ctx->error_handler(error_buf, ctx->user_ctx);
        }
        retval = READSTAT_ERROR_PARSE;
    }

    if (table)
        free(table);
    if (output_buffer)
        free(output_buffer);
    if (error_buf)
        free(error_buf);

    /* suppress warning */
    (void)sav_very_long_string_parse_en_main;

    return retval;
}