warnings

warnings

/*

C wiki parser library and sample application.

Copyright (c) 2010, Mark Wharton
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/
#include <wikiparser.h>
#include <ctype.h>
#include <unistd.h>

#define MAX_HEADINGS WIKI_PARSER_TOKEN_TYPE_HEADING6 - WIKI_PARSER_TOKEN_TYPE_HEADING0

typedef struct Wiki2HTMLUserDataStruct {
	int headings[MAX_HEADINGS];
	bool indexed;
	int lastIndex;
	char* outlink;
	int rowCount;
} Wiki2HTMLUserData, *Wiki2HTMLUserDataPtr;

#define userDataHeadings (((Wiki2HTMLUserDataPtr)userData)->headings)
#define userDataIndexed (((Wiki2HTMLUserDataPtr)userData)->indexed)
#define userDataLastIndex (((Wiki2HTMLUserDataPtr)userData)->lastIndex)
#define userDataOutlink (((Wiki2HTMLUserDataPtr)userData)->outlink)
#define userDataRowCount (((Wiki2HTMLUserDataPtr)userData)->rowCount)

bool echoHeadingID(void *userData, WikiParserBuffer *text)
{
	bool error = false;
	char *id = createWikiParserString(text->data, text->size);
	if (id) {
		int i = 0;
		char *p = id;
		for (; i < text->size; i++) {
			char c = text->data[i];
			if (isalnum(c)) *p++ = tolower(c);
			else if (p > id && *(p - 1) != '-') *p++ = '-';
		}
		*p = '\0';
		error = fprintf(stdout, "%s", id) <= 0;
		wikiParserStringFree(id);
		id = NULL;
	}
	else error = true; // out of memory
	return !error;
}

bool echoHeadingIndex(void *userData, int index)
{
	bool error = false;
	int i = 1; /* skip title */
	for (; i < index && !error; i++) {
		error = fprintf(stdout, "%d.", userDataHeadings[i]) <= 0;
	}
	for (; i < MAX_HEADINGS; i++) {
		userDataHeadings[i] = 0;
	}
	return !error;
}

%%{
	machine EscapeTextWriter;
	write data;
}%%

#pragma unused (EscapeTextWriter_en_main,EscapeTextWriter_error)

bool escapeTextWriter(void *userData, WikiParserBuffer *text)
{
	assert(text);
	bool error = false;
	const char *p = text->data;
	const char *pe = text->data + text->size;
	const char *eof = pe;
	int act;
	int cs;
	const char *ts;
	const char *te;

	%%{
		main := |*
			'&' { if ((error = fprintf(stdout, "&" ) <= 0)) fbreak; };
			'<' { if ((error = fprintf(stdout, "<"  ) <= 0)) fbreak; };
			'>' { if ((error = fprintf(stdout, ">"  ) <= 0)) fbreak; };
			'"' { if ((error = fprintf(stdout, """) <= 0)) fbreak; };
			[^&<>""]+ { if ((error = fwrite(ts, te - ts, 1, stdout) <= 0)) fbreak; };
		*|;

		# Initialize and execute.
		write init;
		write exec;
	}%%

	if (!error) error = cs < EscapeTextWriter_first_final;
	return !error;
}

bool wikiReader(void *userData, WikiParserBuffer *buffer, size_t *size) {
	*size = fread((char *)buffer->data, 1, buffer->size, stdin);
	return ferror(stdin) == 0;
}

bool wikiTitleWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_HEADING1:
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			if (userDataHeadings[i] == 0) {
				if (!error) error = !escapeTextWriter(userData, token->text);
				userDataHeadings[i] = 1;
			}
			break;
		}
		default:
			break;
	}
	return !error;
}

bool wikiTOCWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_HEADING0: /* Not used directly; for calculations. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING1: /* title */
		case WIKI_PARSER_TOKEN_TYPE_HEADING2: /* 1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING3: /* 1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING4: /* 1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING5: /* 1.1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING6: /* 1.1.1.1.1. */
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			userDataHeadings[i]++;
			if (i) {
				i = index - userDataLastIndex;
				if (i < 0) {
					i = -i;
					while (i-- && !error) error = fprintf(stdout, "
") <= 0;
				}
				else if (i > 0) while (i-- && !error) error = fprintf(stdout, "
") <= 0;
				if (!error) error = fprintf(stdout, "
<a href=\"#") <= 0;
				if (!error) error = !echoHeadingIndex(userData, index);
				if (!error) error = fprintf(stdout, "\">") <= 0;
				if (userDataIndexed) {
					if (!error) error = !echoHeadingIndex(userData, index);
					if (!error) error = fprintf(stdout, " ") <= 0;
				}
				if (!error) error = !escapeTextWriter(userData, token->text);
				if (!error) error = fprintf(stdout, "
") <= 0;
			}
			userDataLastIndex = index;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_NONE:
		{
			int index = 1, i;
			i = index - userDataLastIndex;
			if (i < 0) {
				i = -i;
				while (i-- && !error) error = fprintf(stdout, "
") <= 0;
			}
			break;
		}
		default:
			break;
	}
	return !error;
}

bool wikiWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_BOLD:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_CODE:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONDESC:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONTERM:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_HEADING0: /* Not used directly; for calculations. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING1: /* title */
		case WIKI_PARSER_TOKEN_TYPE_HEADING2: /* 1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING3: /* 1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING4: /* 1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING5: /* 1.1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING6: /* 1.1.1.1.1. */
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			userDataHeadings[i]++;
			if (!error) error = fprintf(stdout, "<h%d id=\"", index) <= 0;
			if (!error) error = !echoHeadingIndex(userData, index);
			if (!error) error = fprintf(stdout, "\">") <= 0;
			if (!error) error = fprintf(stdout, "<a id=\"") <= 0;
			if (!error) error = !echoHeadingID(userData, token->text);
			if (!error) error = fprintf(stdout, "\" />") <= 0;
			if (userDataIndexed) {
				if (!error) error = !echoHeadingIndex(userData, index);
				if (!error) error = fprintf(stdout, " ") <= 0;
			}
			if (!error) error = !escapeTextWriter(userData, token->text);
			if (!error) error = fprintf(stdout, "</h%d>", index) <= 0;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_HORIZONTALRULE:
			error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_IMAGE:
			if (!error) error = fprintf(stdout, "<img src=\"") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->source);
			if (token->text) {
				if (!error) error = fprintf(stdout, "\" alt=\"") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->text);
			}
			if (!error) error = fprintf(stdout, "\" />") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_ITALIC:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_KEYBOARD:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LINEBREAK:
			error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LINK:
			if (!error) error = fprintf(stdout, "<a href=\"") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->target);
			if (token->target->size == 1 && *token->target->data == '#') { /* Page reference text. */
				if (!error) error = !echoHeadingID(userData, token->text ? token->text : token->target);
			}
			if (!error) error = fprintf(stdout, "\">") <= 0;
			if (token->source) {
				// image inside links hack
				if (!error) error = fprintf(stdout, "<img src=\"") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->source);
				if (!error) error = fprintf(stdout, "\" />") <= 0;
			}
			else {
				if (!error) error = !escapeTextWriter(userData, token->text ? token->text : token->target);
			}
			if (userDataOutlink && *userDataOutlink && (strncmp(token->target->data, "http", 4) == 0 || strncmp(token->target->data, "./", 2) == 0)) { /* HTML markup for outlink. */
				if (!error) error = fprintf(stdout, "%s", userDataOutlink) <= 0;
			}
			if (!error) error = fprintf(stdout, "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LISTITEM:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_ORDEREDLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_PARAGRAPH:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_PREFORMATTED:
			if (!error) error = fprintf(stdout, "
") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->text);
			if (!error) error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SAMPLECODE:
			error = fprintf(stdout, token->state ? "<samp>" : "</samp>") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SUBSCRIPT:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SUPERSCRIPT:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLE:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			userDataRowCount = 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEDATA:
			error = fprintf(stdout, token->state ? "" : "	") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEHEADER:
			error = fprintf(stdout, token->state ? "" : "	") <= 0;
			userDataRowCount = 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEROW:
		{
			bool alt = userDataRowCount % 2; /* User data is used to keep track of the row state. */
			error = fprintf(stdout, token->state ? alt ? "
" : "
" : "
") <= 0;
			if (token->state) userDataRowCount += 1;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_TEXT:
			error = !escapeTextWriter(userData, token->text);
			break;
		case WIKI_PARSER_TOKEN_TYPE_UNDERLINE:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_UNORDEREDLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_VARIABLE:
			error = fprintf(stdout, token->state ? "<var>" : "</var>") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKINOWIKI:
			if (token->state) { /* Render in monospace. */
				if (!error) error = fprintf(stdout, "") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->text);
				if (!error) error = fprintf(stdout, "") <= 0;
			} else {
				if (!error) error = !escapeTextWriter(userData, token->text);
			}
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKIPLACEHOLDER:
			if (!error) error = fprintf(stdout, "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKIPLUGIN:
		{
			char *name = createWikiParserString(token->text->data, token->text->size);
			if (name && *name == '$') {
				char *value = getenv(name + 1);
				if (value && *value && !error) error = fprintf(stdout, "%s", value) <= 0;
				wikiParserStringFree(name);
				name = NULL;
			}
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_NONE:
			/* end of stream */
			break;
		default:
			error = true;
	}
	return !error;
}

int main(int argc, char **argv)
{
	bool aflag = false;
	bool mflag = false;
	bool xflag = false;
	char *bvalue = NULL;
	char *ovalue = NULL;
	WriteTokenFunc writer = wikiWriter;
	int c;
	while ((c = getopt(argc, argv, "ab:mo:w:x")) != -1) {
		switch (c) {
			case 'a': /* Wiki creole additions flag. */
				aflag = true;
				break;
			case 'b': /* Input stream buffer size for testing. */
				bvalue = optarg;
				break;
			case 'm': /* Inline nowiki monospaced flag. */
				mflag = true;
				break;
			case 'o': /* HTML markup for outlink. */
				ovalue = optarg;
				break;
			case 'w': /* Special token writer. */
				if (strcmp(optarg, "title") == 0) writer = wikiTitleWriter;
				else if (strcmp(optarg, "toc") == 0) writer = wikiTOCWriter;
				else fprintf(stderr, "Error: Unknown writer %s\n", optarg);
				break;
			case 'x': /* Indexed content Headings and TOC labels. */
				xflag = true;
				break;
			case '?':
				/* fall through */
			default:
				fprintf(stderr, "Usage: %s [-a] [-b bufferSize] [-m] [-o outlink] [-w writer] [-x]\n", argv[0]);
				return 1;
		}
	}
	WikiParserConfig config = { aflag, false, mflag, NULL, writer };
	WikiParser parser = createWikiParser(&config);
	if (parser) {
		Wiki2HTMLUserData userData;
		int i = MAX_HEADINGS;
		while (i--) userData.headings[i] = 0;
		userData.indexed = xflag;
		userData.lastIndex = 0;
		userData.outlink = ovalue;
		wikiParserSetUserData(parser, &userData);
		size_t size = bvalue ? atoi(bvalue) : WIKI_PARSER_BUFFER_SIZE;
		WikiParserBuffer *buffer = createWikiParserBuffer(size);
		if (buffer) {
			if (!wikiParserParseStream(parser, buffer, wikiReader, 0)) {
				fprintf(stderr, "Error: parser error: %d %s (line %d)\n",
						wikiParserGetErrorCode(parser), wikiParserGetErrorString(parser),
						wikiParserGetCurrentLine(parser));
				return 1;
			}
			wikiParserBufferFree(buffer);
			buffer = NULL;
		} else {
			fprintf(stderr, "Error: could not allocate buffer: %lld\n", (long long)buffer->size);
			return 1;
		}
		wikiParserFree(parser);
		parser = NULL;
	} else {
		fprintf(stderr, "Error: could not create parser\n");
		return 1;
	}
	return 0;
}

/*

C wiki parser library and sample application.

Copyright (c) 2010, Mark Wharton
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/
#include <wikiparser.h>
#include <ctype.h>
#include <unistd.h>

#define MAX_HEADINGS WIKI_PARSER_TOKEN_TYPE_HEADING6 - WIKI_PARSER_TOKEN_TYPE_HEADING0

typedef struct Wiki2HTMLUserDataStruct {
	int headings[MAX_HEADINGS];
	bool indexed;
	int lastIndex;
	char* outlink;
	int rowCount;
} Wiki2HTMLUserData, *Wiki2HTMLUserDataPtr;

#define userDataHeadings (((Wiki2HTMLUserDataPtr)userData)->headings)
#define userDataIndexed (((Wiki2HTMLUserDataPtr)userData)->indexed)
#define userDataLastIndex (((Wiki2HTMLUserDataPtr)userData)->lastIndex)
#define userDataOutlink (((Wiki2HTMLUserDataPtr)userData)->outlink)
#define userDataRowCount (((Wiki2HTMLUserDataPtr)userData)->rowCount)

bool echoHeadingID(void *userData, WikiParserBuffer *text)
{
	bool error = false;
	char *id = createWikiParserString(text->data, text->size);
	if (id) {
		int i = 0;
		char *p = id;
		for (; i < text->size; i++) {
			char c = text->data[i];
			if (isalnum(c)) *p++ = tolower(c);
			else if (p > id && *(p - 1) != '-') *p++ = '-';
		}
		*p = '\0';
		error = fprintf(stdout, "%s", id) <= 0;
		wikiParserStringFree(id);
		id = NULL;
	}
	else error = true; // out of memory
	return !error;
}

bool echoHeadingIndex(void *userData, int index)
{
	bool error = false;
	int i = 1; /* skip title */
	for (; i < index && !error; i++) {
		error = fprintf(stdout, "%d.", userDataHeadings[i]) <= 0;
	}
	for (; i < MAX_HEADINGS; i++) {
		userDataHeadings[i] = 0;
	}
	return !error;
}

%%{
	machine EscapeTextWriter;
	write data;
}%%

bool escapeTextWriter(void *userData, WikiParserBuffer *text)
{
	assert(text);
	bool error = false;
	const char *p = text->data;
	const char *pe = text->data + text->size;
	const char *eof = pe;
	int act;
	int cs;
	const char *ts;
	const char *te;

	%%{
		main := |*
			'&' { if ((error = fprintf(stdout, "&" ) <= 0)) fbreak; };
			'<' { if ((error = fprintf(stdout, "<"  ) <= 0)) fbreak; };
			'>' { if ((error = fprintf(stdout, ">"  ) <= 0)) fbreak; };
			'"' { if ((error = fprintf(stdout, """) <= 0)) fbreak; };
			[^&<>""]+ { if ((error = fwrite(ts, te - ts, 1, stdout) <= 0)) fbreak; };
		*|;

		# Initialize and execute.
		write init;
		write exec;
	}%%

	if (!error) error = cs < EscapeTextWriter_first_final;
	return !error;
}

bool wikiReader(void *userData, WikiParserBuffer *buffer, size_t *size) {
	*size = fread((char *)buffer->data, 1, buffer->size, stdin);
	return ferror(stdin) == 0;
}

bool wikiTitleWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_HEADING1:
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			if (userDataHeadings[i] == 0) {
				if (!error) error = !escapeTextWriter(userData, token->text);
				userDataHeadings[i] = 1;
			}
			break;
		}
		default:
			break;
	}
	return !error;
}

bool wikiTOCWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_HEADING0: /* Not used directly; for calculations. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING1: /* title */
		case WIKI_PARSER_TOKEN_TYPE_HEADING2: /* 1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING3: /* 1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING4: /* 1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING5: /* 1.1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING6: /* 1.1.1.1.1. */
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			userDataHeadings[i]++;
			if (i) {
				i = index - userDataLastIndex;
				if (i < 0) {
					i = -i;
					while (i-- && !error) error = fprintf(stdout, "
") <= 0;
				}
				else if (i > 0) while (i-- && !error) error = fprintf(stdout, "
") <= 0;
				if (!error) error = fprintf(stdout, "
<a href=\"#") <= 0;
				if (!error) error = !echoHeadingIndex(userData, index);
				if (!error) error = fprintf(stdout, "\">") <= 0;
				if (userDataIndexed) {
					if (!error) error = !echoHeadingIndex(userData, index);
					if (!error) error = fprintf(stdout, " ") <= 0;
				}
				if (!error) error = !escapeTextWriter(userData, token->text);
				if (!error) error = fprintf(stdout, "
") <= 0;
			}
			userDataLastIndex = index;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_NONE:
		{
			int index = 1, i;
			i = index - userDataLastIndex;
			if (i < 0) {
				i = -i;
				while (i-- && !error) error = fprintf(stdout, "
") <= 0;
			}
			break;
		}
		default:
			break;
	}
	return !error;
}

bool wikiWriter(void *userData, WikiParserToken *token)
{
	assert(token);
	bool error = false;
	switch (token->type) {
		case WIKI_PARSER_TOKEN_TYPE_BOLD:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_CODE:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONDESC:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_DEFINITIONTERM:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_HEADING0: /* Not used directly; for calculations. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING1: /* title */
		case WIKI_PARSER_TOKEN_TYPE_HEADING2: /* 1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING3: /* 1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING4: /* 1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING5: /* 1.1.1.1. */
		case WIKI_PARSER_TOKEN_TYPE_HEADING6: /* 1.1.1.1.1. */
		{
			int index = token->type - WIKI_PARSER_TOKEN_TYPE_HEADING0;
			int i = index - 1;
			userDataHeadings[i]++;
			if (!error) error = fprintf(stdout, "<h%d id=\"", index) <= 0;
			if (!error) error = !echoHeadingIndex(userData, index);
			if (!error) error = fprintf(stdout, "\">") <= 0;
			if (!error) error = fprintf(stdout, "<a id=\"") <= 0;
			if (!error) error = !echoHeadingID(userData, token->text);
			if (!error) error = fprintf(stdout, "\" />") <= 0;
			if (userDataIndexed) {
				if (!error) error = !echoHeadingIndex(userData, index);
				if (!error) error = fprintf(stdout, " ") <= 0;
			}
			if (!error) error = !escapeTextWriter(userData, token->text);
			if (!error) error = fprintf(stdout, "</h%d>", index) <= 0;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_HORIZONTALRULE:
			error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_IMAGE:
			if (!error) error = fprintf(stdout, "<img src=\"") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->source);
			if (token->text) {
				if (!error) error = fprintf(stdout, "\" alt=\"") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->text);
			}
			if (!error) error = fprintf(stdout, "\" />") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_ITALIC:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_KEYBOARD:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LINEBREAK:
			error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LINK:
			if (!error) error = fprintf(stdout, "<a href=\"") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->target);
			if (token->target->size == 1 && *token->target->data == '#') { /* Page reference text. */
				if (!error) error = !echoHeadingID(userData, token->text ? token->text : token->target);
			}
			if (!error) error = fprintf(stdout, "\">") <= 0;
			if (token->source) {
				// image inside links hack
				if (!error) error = fprintf(stdout, "<img src=\"") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->source);
				if (!error) error = fprintf(stdout, "\" />") <= 0;
			}
			else {
				if (!error) error = !escapeTextWriter(userData, token->text ? token->text : token->target);
			}
			if (userDataOutlink && *userDataOutlink && (strncmp(token->target->data, "http", 4) == 0 || strncmp(token->target->data, "./", 2) == 0)) { /* HTML markup for outlink. */
				if (!error) error = fprintf(stdout, "%s", userDataOutlink) <= 0;
			}
			if (!error) error = fprintf(stdout, "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_LISTITEM:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_ORDEREDLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_PARAGRAPH:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_PREFORMATTED:
			if (!error) error = fprintf(stdout, "
") <= 0;
			if (!error) error = !escapeTextWriter(userData, token->text);
			if (!error) error = fprintf(stdout, "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SAMPLECODE:
			error = fprintf(stdout, token->state ? "<samp>" : "</samp>") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SUBSCRIPT:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_SUPERSCRIPT:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLE:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			userDataRowCount = 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEDATA:
			error = fprintf(stdout, token->state ? "" : "	") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEHEADER:
			error = fprintf(stdout, token->state ? "" : "	") <= 0;
			userDataRowCount = 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_TABLEROW:
		{
			bool alt = userDataRowCount % 2; /* User data is used to keep track of the row state. */
			error = fprintf(stdout, token->state ? alt ? "
" : "
" : "
") <= 0;
			if (token->state) userDataRowCount += 1;
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_TEXT:
			error = !escapeTextWriter(userData, token->text);
			break;
		case WIKI_PARSER_TOKEN_TYPE_UNDERLINE:
			error = fprintf(stdout, token->state ? "" : "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_UNORDEREDLIST:
			error = fprintf(stdout, token->state ? "
" : "
") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_VARIABLE:
			error = fprintf(stdout, token->state ? "<var>" : "</var>") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKINOWIKI:
			if (token->state) { /* Render in monospace. */
				if (!error) error = fprintf(stdout, "") <= 0;
				if (!error) error = !escapeTextWriter(userData, token->text);
				if (!error) error = fprintf(stdout, "") <= 0;
			} else {
				if (!error) error = !escapeTextWriter(userData, token->text);
			}
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKIPLACEHOLDER:
			if (!error) error = fprintf(stdout, "") <= 0;
			break;
		case WIKI_PARSER_TOKEN_TYPE_WIKIPLUGIN:
		{
			char *name = createWikiParserString(token->text->data, token->text->size);
			if (name && *name == '$') {
				char *value = getenv(name + 1);
				if (value && *value && !error) error = fprintf(stdout, "%s", value) <= 0;
				wikiParserStringFree(name);
				name = NULL;
			}
			break;
		}
		case WIKI_PARSER_TOKEN_TYPE_NONE:
			/* end of stream */
			break;
		default:
			error = true;
	}
	return !error;
}

int main(int argc, char **argv)
{
	bool aflag = false;
	bool mflag = false;
	bool xflag = false;
	char *bvalue = NULL;
	char *ovalue = NULL;
	WriteTokenFunc writer = wikiWriter;
	int c;
	while ((c = getopt(argc, argv, "ab:mo:w:x")) != -1) {
		switch (c) {
			case 'a': /* Wiki creole additions flag. */
				aflag = true;
				break;
			case 'b': /* Input stream buffer size for testing. */
				bvalue = optarg;
				break;
			case 'm': /* Inline nowiki monospaced flag. */
				mflag = true;
				break;
			case 'o': /* HTML markup for outlink. */
				ovalue = optarg;
				break;
			case 'w': /* Special token writer. */
				if (strcmp(optarg, "title") == 0) writer = wikiTitleWriter;
				else if (strcmp(optarg, "toc") == 0) writer = wikiTOCWriter;
				else fprintf(stderr, "Error: Unknown writer %s\n", optarg);
				break;
			case 'x': /* Indexed content Headings and TOC labels. */
				xflag = true;
				break;
			case '?':
				/* fall through */
			default:
				fprintf(stderr, "Usage: %s [-a] [-b bufferSize] [-m] [-o outlink] [-w writer] [-x]\n", argv[0]);
				return 1;
		}
	}
	WikiParserConfig config = { aflag, false, mflag, NULL, writer };
	WikiParser parser = createWikiParser(&config);
	if (parser) {
		Wiki2HTMLUserData userData;
		int i = MAX_HEADINGS;
		while (i--) userData.headings[i] = 0;
		userData.indexed = xflag;
		userData.lastIndex = 0;
		userData.outlink = ovalue;
		wikiParserSetUserData(parser, &userData);
		size_t size = bvalue ? atoi(bvalue) : WIKI_PARSER_BUFFER_SIZE;
		WikiParserBuffer *buffer = createWikiParserBuffer(size);
		if (buffer) {
			if (!wikiParserParseStream(parser, buffer, wikiReader, 0)) {
				fprintf(stderr, "Error: parser error: %d %s (line %d)\n",
						wikiParserGetErrorCode(parser), wikiParserGetErrorString(parser),
						wikiParserGetCurrentLine(parser));
				return 1;
			}
			wikiParserBufferFree(buffer);
			buffer = NULL;
		} else {
			fprintf(stderr, "Error: could not allocate buffer: %lld\n", (long long)buffer->size);
			return 1;
		}
		wikiParserFree(parser);
		parser = NULL;
	} else {
		fprintf(stderr, "Error: could not create parser\n");
		return 1;
	}
	return 0;
}


add comments to Parser::throw_unexpected

#include "hext/parser.h"

namespace hext {

ParseError::ParseError(const std::string& msg)
: std::runtime_error(msg)
{
}

Parser::Parser(const char * begin, const char * end)
: p_begin_(begin),
  p(begin),
  pe(end),
  eof(end),
  cs(0)
{
  using namespace ragel;
  %%{
    machine hext;
    include "hext.rl";
    write init;
  }%%
}

std::vector<Rule> Parser::parse()
{
  using namespace ragel;

  RuleBuilder rule;
  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";
  bool rule_start = false;

  %% write exec;

  return rule.get_rules_and_reset();
}

void Parser::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);

  // The line and char offset of the unexpected character
  CharPosPair pos =
    GetCharPosition(this->p, this->p_begin_, this->pe);

  // GetCharPosition's offsets are zero-based.
  const auto line_count = pos.first + 1;
  const auto char_count = pos.second + 1;

  std::string char_name;
  if( this->p == this->pe )
    char_name = "[eof]";
  else
    char_name = GetCharName(*this->p);

  std::stringstream error_msg;
  error_msg << "Unexpected character '"
            << char_name
            << "' at line "
            << line_count
            << ", char "
            << char_count
            << "\n\n";

  // We need a 'past-the-last-element' iterator.
  const char * end = this->p;

  // this->p points to the unexpected character. We want to include this
  // character when printing error location, unless it is a newline, then it
  // would just mess up the output.
  if( this->p < this->pe && *this->p != '\n' )
    end++;

  // The amount of chars needed to print the biggest line number.
  int number_width = GetDecNumberWidth(line_count);

  PrintWithLineNumbers(this->p_begin_, end, number_width, error_msg);

  // Print a visual indicator right under the unexpected character.
  // We need to know the amount of indentation required.
  // PrintWithLineNumbers prints lines like this:
  // 1: An SQL query goes into a bar,
  // 2: walks up to two tables and asks,
  // 3: 'Can I join you?'
  size_t indent = number_width // chars required to print the line number
                + 2            // ": "
                + pos.second;  // offset of the unexpected character from the
                               // beginning of the line.

  error_msg << std::string(indent, ' ')
            << "^ here";

  throw ParseError(error_msg.str());
}

void Parser::throw_unknown_token(
  const std::string& token_name,
  const std::string& token
) const
{
  // See Parser::throw_unexpected for some commentary.

  assert(this->p && this->p_begin_ && this->pe);

  CharPosPair pos =
    GetCharPosition(this->p, this->p_begin_, this->pe);

  const auto line_count = pos.first + 1;
  const auto char_count = pos.second + 1;

  std::stringstream error_msg;
  error_msg << "Unknown "
            << token_name
            << " '"
            << token
            << "' at line "
            << line_count
            << ", char "
            << char_count
            << "\n\n";

  const char * end = this->p;
  if( this->p < this->pe )
    end++;

  int number_width = GetDecNumberWidth(line_count);

  PrintWithLineNumbers(this->p_begin_, end, number_width, error_msg);

  size_t indent = number_width
                + 2
                + pos.second
                - token.size();

  error_msg << std::string(indent, ' ')
            << std::string(token.size(), '^')
            << " here";

  throw ParseError(error_msg.str());
}

} // namespace hext

#include "hext/parser.h"

namespace hext {

ParseError::ParseError(const std::string& msg)
: std::runtime_error(msg)
{
}

Parser::Parser(const char * begin, const char * end)
: p_begin_(begin),
  p(begin),
  pe(end),
  eof(end),
  cs(0)
{
  using namespace ragel;
  %%{
    machine hext;
    include "hext.rl";
    write init;
  }%%
}

std::vector<Rule> Parser::parse()
{
  using namespace ragel;

  RuleBuilder rule;
  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";
  bool rule_start = false;

  %% write exec;

  return rule.get_rules_and_reset();
}

void Parser::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);

  CharPosPair pos =
    GetCharPosition(this->p, this->p_begin_, this->pe);

  const auto line_count = pos.first + 1;
  const auto char_count = pos.second + 1;

  std::string char_name;
  if( this->p == this->pe )
    char_name = "[eof]";
  else
    char_name = GetCharName(*this->p);

  const char * end = this->p;
  if( this->p < this->pe )
    end++;

  std::stringstream error_msg;
  error_msg << "Unexpected character '"
            << char_name
            << "' at line "
            << line_count
            << ", char "
            << char_count
            << "\n\n";

  int number_width = GetDecNumberWidth(line_count);
  PrintWithLineNumbers(this->p_begin_, end, number_width, error_msg);

  size_t indent = pos.second
                + 2 /* strlen(": ") */
                + number_width;
  error_msg << std::string(indent, ' ')
            << "^ here";

  throw ParseError(error_msg.str());
}

void Parser::throw_unknown_token(
  const std::string& token_name,
  const std::string& token
) const
{
  assert(this->p && this->p_begin_ && this->pe);

  CharPosPair pos =
    GetCharPosition(this->p, this->p_begin_, this->pe);

  const auto line_count = pos.first + 1;
  const auto char_count = pos.second + 1;

  std::stringstream error_msg;
  error_msg << "Unknown "
            << token_name
            << " '"
            << token
            << "' at line "
            << line_count
            << ", char "
            << char_count
            << "\n\n";

  const char * end = this->p;
  if( this->p < this->pe )
    end++;

  int number_width = GetDecNumberWidth(line_count);
  PrintWithLineNumbers(this->p_begin_, end, number_width, error_msg);

  size_t indent = pos.second
                + 2 /* strlen(": ") */
                + number_width
                - token.size();
  error_msg << std::string(indent, ' ')
            << std::string(token.size(), '^')
            << " here";

  throw ParseError(error_msg.str());
}

} // namespace hext


Make it less attractive to create methods depending on shared state

At the moment, the parser can be used concurrently. If we use @vars, for
example, this will not be possible.

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action write_command { message.command = data[m..p] }

  action store_key { key = data[m..p] }
  action write_header { message.headers[key] = data[m..p] }

  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;
  OCTET = any;
  HEADER_OCTET = OCTET - CR - LF - ":";

  command_name = "CONNECT";
  command = command_name > mark @ write_command;

  header_key = HEADER_OCTET+ > mark @ store_key;
  header_value = HEADER_OCTET* > mark @ write_header;
  header = header_key ":" header_value EOL;

  message := command EOL header* EOL NULL;
}%%

module Stompede
  module Stomp
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @return [Stomp::Message]
      def self.parse(data)
        # re-encode the input as BINARY to be able to refer
        # on the byte-level with data[i].ord
        data = data.force_encoding("BINARY")

        # this is where our parsed components end up
        message = Stomp::Message.new

        p = 0 # pointer to current character
        pe = data.length # pointer to end of input
        cs = Stomp::Parser.start # current starting state
        m = 0 # pointer to marked character (for buffering)

        # write out the ragel state machine parser
        %% write exec;

        # if parsing parsed a complete message, return it
        message if cs >= Stomp::Parser.first_final
      end
    end
  end
end

%%{
  machine Message;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action write_command { message.command = data[m..p] }

  action store_key { key = data[m..p] }
  action write_header { message.headers[key] = data[m..p] }

  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;
  OCTET = any;
  HEADER_OCTET = OCTET - CR - LF - ":";

  command_name = "CONNECT";
  command = command_name > mark @ write_command;

  header_key = HEADER_OCTET+ > mark @ store_key;
  header_value = HEADER_OCTET* > mark @ write_header;
  header = header_key ":" header_value EOL;

  message := command EOL header* EOL NULL;
}%%

module Stompede
  module Stomp
    module Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @param [String] data
        # @return [Stomp::Message]
        def parse(data)
          # re-encode the input as BINARY to be able to refer
          # on the byte-level with data[i].ord
          data = data.force_encoding("BINARY")

          # this is where our parsed components end up
          message = Stomp::Message.new

          p = 0 # pointer to current character
          m = 0 # pointer to marked character (for buffering)
          pe = data.length # pointer to end of input
          cs = start # current starting state

          # write out the ragel state machine parser
          %% write exec;

          # if parsing parsed a complete message, return it
          message if cs >= first_final
        end
      end
    end
  end
end


n-level nesting

n-level nesting

%%{
  machine params_parser;
  action H { @head = p }
  action T { @tail = p }
  action inject_key { @hash[key] = {} }

  DELIMITER = (([, ])+ | '\n')                      >H@T @{ log_event(:delimiter) };
  LEVEL = ('(')+                                    >H@T @{ log_event(:level) };
  END_LEVEL = (')')+                                >H@T @{ log_event(:end_level); p + 1 == pe and @hash[key] = {}; };
  INPUT = (any - LEVEL - END_LEVEL - DELIMITER)+    >H@T @{ log_event(:input); memo_key; };

  main := (
    INPUT
    (DELIMITER @{ @hash[key] = {}; fnext main; } | END_LEVEL DELIMITER @{ @hash[key] = {}; @hash = @hash_stack.pop; fret; } | LEVEL @{ @hash[key] = {}; @hash_stack.push @hash; @hash = @hash[key]; fcall main; })
  )*;
}%%

module ParamsTree
  class Parser
    attr_accessor :data

    def initialize
      @events = []
      @hash = {}
      @hash_stack = []
      @key = nil
      %% write data;
    end

    def process(input)
      @data = input.unpack('c*')
      stack = []

      %% write init;
      %% write exec;

      #puts @events.inspect
      puts @hash_stack.inspect
    end

    def char
      @data[@head..@tail]
    end

    def strokes
      char.pack('c*')
    end

    def log_event(token)
      @events << { token => strokes }
    end

    def memo_key
      @key = char
    end

    def key
      @key.pack('c*')
    end
  end
end

ParamsTree::Parser.new.process "default(all,usadasdid(zczxc(pio),cvb)),isdsdd(wow,er),sed(rew,tre,yrt)"

%%{
  machine params_parser;
  action H { @head = p }
  action T { @tail = p }
  action inject_key { @hash[key] = {} }

  EOL = ('\n' | '\r\n');
  DELIMITER = ([, ])+                            >H@T @{ log_event(:delimiter) };
  LEVEL = [(]                                    >H@T @{ log_event(:level) };
  END_LEVEL = [)]                                >H@T @{ log_event(:end_level) };
  INPUT = (any - LEVEL - END_LEVEL - DELIMITER)+ >H@T @{ log_event(:input); memo_key; };

  main := (
    INPUT?
    (DELIMITER @{ fnext main; } | LEVEL @{ @hash[key] = {}; @parent = @hash; @hash = @hash[key]; fcall nested; })
  )*;

  nested := (
    INPUT
    (DELIMITER @{ @hash[key] = {}; fnext nested; } | END_LEVEL @{ @hash[key] = {}; @hash = @parent; fret; })
  );
}%%

module ParamsTree
  class Parser
    attr_accessor :data

    def initialize
      @events = []
      @hash = {}
      @parent = {}
      @key = nil
      %% write data;
    end

    def process(input)
      @data = input.unpack('c*')
      stack = []

      %% write init;
      %% write exec;

      puts @events.inspect
      puts key
      puts @hash.inspect
      puts @parent.inspect
    end

    def char
      @data[@head..@tail]
    end

    def strokes
      char.pack('c*')
    end

    def log_event(token)
      @events << { token => strokes }
    end

    def memo_key
      @key = char
    end

    def key
      @key.pack('c*')
    end
  end
end

ParamsTree::Parser.new.process "default(all,usadasdid),isdsdd(wow,er),sed(rew,tre,yrt)"


Make parser byte lookup and data slicing faster

%%{
  machine message;

  getkey data.getbyte(p); # code for retrieving current char

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action mark_key { mk = data.byteslice(m, p) }

  action write_command { message.write_command(data.byteslice(m, p)) }
  action write_header { message.write_header(mk, data.byteslice(m, p)) }
  action write_body { message.write_body(data.byteslice(m, p)) }

  action init_message {
    message = Stomp::Message.new
  }
  action finish_headers {}
  action finish_message {
    return message
  }

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark % mark_key;
  header_value = HEADER_OCTET* > mark;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) % write_body :> NULL;

  message := (command > init_message) :> headers :> (dynamic_body @ finish_message);
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.length # end of chunk
        eof = :ignored # end of input

        p = state.cursor # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        m = state.mark # pointer to marked character (for data buffering)
        mk = state.mark_key # key for header currently being read

        %% write exec;

        state.cursor = p
        state.message = message
        state.current_state = cs
        state.mark = m
        state.mark_key = mk

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size

        @cursor = 0
        @current_state = Stomp::Parser.start
        @message = nil
        @mark = 0
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [Integer] index of parsing cursor in data
      attr_accessor :cursor

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark_key

      # @return [Integer] index of parsing cursor in data
      # @return [Integer] index of parsing cursor in data
      def parse(data)
        Parser.parse(data, self)
      end
    end
  end
end

%%{
  machine message;

  getkey data[p].ord; # code for retrieving current char (default)

  # data, p, pe, eof, cs, top, stack, ts, te and act

  action mark { m = p }
  action mark_key { mk = data[m...p] }

  action write_command { message.write_command(data[m...p]) }
  action write_header { message.write_header(mk, data[m...p]) }
  action write_body { message.write_body(data[m...p]) }

  action init_message {
    message = Stomp::Message.new
  }
  action finish_headers {}
  action finish_message {
    return message
  }

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark % mark_key;
  header_value = HEADER_OCTET* > mark;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  dynamic_body = (OCTET* > mark) % write_body :> NULL;

  message := (command > init_message) :> headers :> (dynamic_body @ finish_message);
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.length # end of chunk
        eof = :ignored # end of input

        p = state.cursor # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        m = state.mark # pointer to marked character (for data buffering)
        mk = state.mark_key # key for header currently being read

        %% write exec;

        state.cursor = p
        state.message = message
        state.current_state = cs
        state.mark = m
        state.mark_key = mk

        nil
      end

      # Construct the parser.
      #
      # The buffer_size parameter determines how big the largest
      # chunk of data may become. This includes commands, header
      # keys, header values, and the body. If this size is reached,
      # the parser will throw a #{BufferLimitExceeded} error.
      #
      # The message_size parameter determines how big the largest
      # message may become. This includes all of the content in the
      # message being parsed. If this size is reached for an individual
      # message, the parser will throw a #{MessageSizeExceeded} error.
      #
      # @param [Integer] buffer_size (10K) maximum buffer size
      # @param [Integer] message_size (buffer_size) maximum message size
      def initialize(buffer_size = 1024 * 1024, message_size = buffer_size)
        @buffer_size = buffer_size
        @message_size = message_size

        @cursor = 0
        @current_state = Stomp::Parser.start
        @message = nil
        @mark = 0
        @mark_key = nil
      end

      # @return [Integer] maximum buffer size for parsed values
      attr_reader :buffer_size

      # @return [Integer] maximum message size for parsed messages
      attr_reader :message_size

      # @return [Integer] index of parsing cursor in data
      attr_accessor :cursor

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark

      # @return [Integer] index of parsing cursor in data
      attr_accessor :mark_key

      # @return [Integer] index of parsing cursor in data
      # @return [Integer] index of parsing cursor in data
      def parse(data)
        Parser.parse(data.force_encoding("BINARY"), self)
      end
    end
  end
end


Allow extra spaces at the end of the command string

Summary:
Open source compatibility.
Memcached handles these gracefully and there are some non-compliant clients out there that issue commands with extra spaces before `\r\n`.
Let's just accept those.

Test Plan:
unit tests
manually tested a few commands

Reviewed By: stepan@fb.com

Subscribers: trunkagent

FB internal diff: D1492373

/* -*- mode: c; -*- */
/* ascii_client.c is built from ascii_client.rl using ragel */

#include <ctype.h>

#include "mcrouter/lib/fbi/debug.h"
#include "mcrouter/lib/mc/_protocol.h"
#include "mcrouter/lib/mc/parser.h"
#include "mcrouter/lib/mc/msg.h"
#include "mcrouter/lib/mc/util.h"

// In libmc.client, only read the first N stats for simplicity of
// implementation. In the future it might make sense to allow an
// arbitrary number of stats, but for now that makes the code far
// easier
#define MAX_NUM_STATS 20

// Are we in the middle of a key being skipped?
#define skipping_key(parser) ((parser)->in_skipped_key &&       \
                              !(parser)->record_skip_key)

%%{

# input file for ragel (http://www.complang.org/ragel/)

# this machine name, which must come first, is used as a prefix for variables
machine ascii;

# named actions. All the actions (named and unnamed) are executed in the
# context of _on_ascii_rx() where "%% write exec;" is called.
action start_token {
  if (!skipping_key(parser)) {
    parser->partial_token = 1;
    parser->te = parser->tbuf;
    ts = p;
  }
}
action finish_token {
  if (parser->partial_token && !skipping_key(parser)) {
    FBI_ASSERT(ts);
    parser->partial_token = 0;
    if (parser->te && parser->te > parser->tbuf) {
      /* we have something stuffed in our cheeks,
         let's put them together and update ts. */
      FBI_ASSERT(ts <= p);
      size_t n = p - ts;
      if(mc_parser_ensure_tbuf(parser, n) != 0) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      } else {
        strncpy(parser->te, ts, n);
        parser->te += n;
        *parser->te = '\0';
        tlen = parser->te - parser->tbuf;
        ts = parser->tbuf;
      }
    } else {
      tlen = p - ts;
      if (tlen >= MC_TOKEN_MAX_LEN) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      }
    }
  }
}

action record_key {
  if (!skipping_key(parser)) {
    if (tlen > MC_KEY_MAX_LEN_ASCII) {
      parser->bad_key = 1;
    }
    if (parser->bad_key) {
      parser->msg->result = mc_res_bad_key;
    } else {
      if (mc_msg_grow(&parser->msg, tlen + 1,
                      (void**) &parser->msg->key.str) != 0) {
        parser->error = parser_out_of_memory;
        fgoto error_handling;
      }
      parser->msg->key.len = tlen;
      strncpy(parser->msg->key.str, ts, tlen);
      FBI_ASSERT(!memcmp(parser->msg->key.str, ts, tlen));
      parser->msg->key.str[tlen] = '\0';
    }
  }
  parser->in_skipped_key = 0;
  parser->in_key = 0;
  parser->bad_key = 0;
}

action fin {
  FBI_ASSERT(parser->msg != NULL);

  // end of msg, fake eof
  pe = eof = p + 1;
  parser->parser_state = parser_msg_header;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = NULL;
  parser->msg_ready(parser->context, 0, msg);
  nmsgs++;

  mc_parser_cleanup_tbuf(parser);
}

nl = '\r'? '\n';
fin = nl @fin;

# server-reported errors. We should only ever see SERVER_ERROR. If we see
# CLIENT_ERROR then libmc probably didn't do enough error checking before
# sending to memcache in the first place. ERROR is absolutely a bug in libmc.
error_handling := '' >{
  if (parser->msg != NULL) {
    mc_msg_decref(parser->msg);
  }
  parser->msg = NULL;
  parser->te = NULL;
  parser->in_key = 0;
  parser->bad_key = 0;
  fbreak;
};

error_code = ' ' digit+ ' '
  @{
    parser->msg->err_code = strtoll(ts, NULL, 10);
  };

action capture_error_value {
  // Since the token we are parsing includes the \r\n, we need
  // to invoke this code upon entering the DFA's final state.
  // Only the '\r' is stored in the token buffer, so we only
  // need to remove one character.
  tlen--;
  if (mc_msg_grow(&parser->msg, tlen + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.len = tlen;
  memcpy(parser->msg->value.str, ts, tlen);
  parser->msg->value.str[tlen] = '\0';
}

# server error apparently needs to be able to read binary data to support
# TAO, which uses SERVER_ERROR to send the fbid of objects back to the
# client during set requests.  There is no size field, so we need to
# keep parsing the response string until we hit \r\n.  Server error
# 307 is returned by a server that is overloaded or is shutting down;
# it is treated specially so that it can cross the application/transport
# boundary.  (mc_res_busy is an artifact of our use of TCP/IP, of our
# desired to keep the connection open while not accepting requests.
# If there was a reliable request-based protocol it would be less
# necessary.)
server_error = 'SERVER_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    if (parser->msg->err_code == SERVER_ERROR_BUSY) {
      parser->msg->result = mc_res_busy;
    } else {
      parser->msg->result = mc_res_remote_error;
    }
  }
  @fin;

# client error message may contain useful stuff, so we also want to save
# it for the caller
client_error = 'CLIENT_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    parser->msg->result = mc_res_client_error;
  }
  @fin;

command_error = 'ERROR'
  @{
    parser->msg->result = mc_res_local_error;
    parser->msg->value.str = "unexpected ERROR reply";
    parser->msg->value.len = strlen(parser->msg->value.str);
    dbg_error("%s", parser->msg->value.str);
    FBI_ASSERT(0);
  } fin;

error = server_error | client_error | command_error;

# Get request is ready to be passed to the caller
action get_req_is_ready {
  FBI_ASSERT(parser->msg != NULL);

  mc_op_t op = parser->msg->op;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = mc_msg_new(0);
  parser->msg_ready(parser->context, 0, msg);
  if (parser->msg == NULL) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }

  parser->msg->op = op;
}

# building blocks
stored = 'STORED' @{ parser->msg->result = mc_res_stored; };
stale_stored = 'STALE_STORED' @{
  parser->msg->result = mc_res_stalestored;
} fin;
not_stored = 'NOT_STORED' @{ parser->msg->result = mc_res_notstored; };
exists = 'EXISTS' @{ parser->msg->result = mc_res_exists; };
not_found = 'NOT_FOUND' @{ parser->msg->result = mc_res_notfound; };
deleted = 'DELETED' @{ parser->msg->result = mc_res_deleted; };

storage_reply = stored | stale_stored | not_stored | exists | not_found | deleted;
storage = storage_reply fin;
arithmetic_reply = (not_found |
  digit+ >start_token %finish_token %{
    parser->msg->delta = strtoll(ts, NULL, 10);
    parser->msg->result = mc_res_stored;
  } ' '* fin);

key_skip = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 1; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

key = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 0; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

# This is used for stats and exec commands which can have keys with spaces.
multi_token = ' '* (print+ -- ( '\r' | '\n' ))
  >start_token %finish_token %{
  while (tlen > 0 && isspace(ts[tlen-1])) {
    tlen--;
  }
  if (mc_msg_grow(&parser->msg, tlen+1, (void**) &parser->msg->key.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->key.len = tlen;
  strncpy(parser->msg->key.str, ts, tlen);
  parser->msg->key.str[tlen] = '\0';
};

lease_token = ('-'? digit+) >start_token %finish_token %{
  parser->msg->lease_id = strtol(ts, NULL, 10);
};
casid = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoll(ts, NULL, 10);
};
flags = digit+ >start_token %finish_token %{
  parser->msg->flags = strtoll(ts, NULL, 10);
};
exptime = digit+ >start_token %finish_token %{
  parser->msg->exptime = strtol(ts, NULL, 10);
};
delta = digit+ >start_token %finish_token %{
  parser->msg->delta = strtoll(ts, NULL, 10);
};
number = digit+ >start_token %finish_token %{
  parser->msg->number = strtoll(ts, NULL, 10);
};
noreply = 'noreply' %{
  parser->msg->noreply = true;
};

value_bytes = digit+ >start_token %finish_token %{
  /* prepare for receiving data */
  size_t len = parser->resid = strtol(ts, NULL, 10);
  parser->off = 0;

  if (mc_msg_grow(&parser->msg, len + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.str[len] = '\0';
  parser->msg->value.len = len;
};
cas_unique = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoull(ts, NULL, 10);
};
dec_octet = digit{1,3};
ipv4_addr = dec_octet "." dec_octet "." dec_octet "." dec_octet;
ipv6_addr = (":" | xdigit)+ ipv4_addr?;
ip_addr = (ipv4_addr | ipv6_addr) >start_token %finish_token %{
  ts[tlen]='\0';
  parser->msg->ipv = 0;
  if (strchr(ts, ':') == NULL) {
    if (inet_pton(AF_INET, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 4;
  } else {
    if (inet_pton(AF_INET6, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 6;
  }
};
age_unknown = 'unknown' %{ parser->msg->number = -1;};
action rx_data {
  /* the binary data of a specified isn't really regular, so we drop out of
     the state machine and read it, then come back in to finish the
     "\r\nEND\r\n". */
  parser->parser_state = parser_body;
  fbreak;
}

# replies

VALUE = 'VALUE' % { parser->msg->result = mc_res_found; };

hit = VALUE ' '+ key_skip ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
  @rx_data nl;

get_reply = (hit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
lease_get = ((hit |
  ('LVALUE' ' '+ key_skip ' '+ lease_token ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
   @rx_data nl
   @{ parser->msg->result = mc_res_notfound; }))
  'END' fin);
META = 'META' % {parser->msg->result = mc_res_found;};
mhit = META ' '+ key_skip ' '+ 'age:' ' '* (number | age_unknown) ';' ' '*
  'exptime:' ' '* exptime ';' ' '* 'from:' ' '* (ip_addr|'unknown') ';' ' '*
  'is_transient:' ' '* flags ' '* nl;
metaget_reply = (mhit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
reply_set = storage;
reply_add = storage;
reply_replace = storage;
reply_lease_set = storage;
reply_append = storage;
reply_prepend = storage;
reply_cas = storage;
delete_reply = deleted | not_found;

stat_name = (any+ -- (cntrl | space)) >start_token %finish_token
%{
  if (parser->msg->stats == NULL) {
    FBI_ASSERT(parser->msg->number == 0);
    size_t stats_size = sizeof(nstring_t) * 2 * MAX_NUM_STATS;
    if (mc_msg_grow(&parser->msg, stats_size, (void**) &parser->msg->stats)) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    memset(parser->msg->stats, 0, stats_size);
  }

  if (parser->msg->number < MAX_NUM_STATS) {
    parser->msg->number++;

    int idx = (parser->msg->number * 2) - 2;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stat_value = (any++ -- cntrl) > start_token %finish_token
%{
  if (parser->msg->number <= MAX_NUM_STATS) {
    int idx = (parser->msg->number * 2) - 1;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }

    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stats_reply = (('STAT ' @{ parser->msg->result = mc_res_ok; }
                stat_name ' '+ stat_value nl)* 'END' fin);
ok = ('OK' @{ parser->msg->result = mc_res_ok; } fin);
version_reply = ('VERSION ' (any* -- ('\r' | '\n')) >start_token %finish_token
  %{
    // use new_msg because msg stays live if realloc fails and can leak
    mc_msg_t *new_msg = mc_msg_realloc(parser->msg,
                                       parser->msg->_extra_size + tlen + 1);
    if (new_msg == NULL) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg = new_msg;
    parser->msg->result = mc_res_ok;
    char* version = parser->msg->value.str = (char *) &parser->msg[1];
    parser->msg->value.len = tlen;
    strncpy(version, ts, tlen);
    version[tlen] = '\0';
  }
  fin);

reply := get_reply | lease_get | metaget_reply |
        reply_set | reply_add | reply_replace | reply_lease_set |
        reply_append | reply_prepend | reply_cas | delete_reply |
        arithmetic_reply |
        stats_reply | ok | version_reply | error;

# requests

set = 'set' @ { parser->msg->op = mc_op_set; };
add = 'add' @ { parser->msg->op = mc_op_add; };
replace = 'replace' @ { parser->msg->op = mc_op_replace; };
leaseset = 'lease-set' @ { parser->msg->op = mc_op_lease_set; };
append = 'append' @ { parser->msg->op = mc_op_append; };
prepend = 'prepend' @ { parser->msg->op = mc_op_prepend; };

set_op = set | add | replace | append | prepend;

delete_op = 'delete' @ { parser->msg->op = mc_op_delete; };

get_op = 'get' @ { parser->msg->op = mc_op_get; } |
         'gets' @ {parser->msg->op = mc_op_gets; } |
         'lease-get' @ { parser->msg->op = mc_op_lease_get; } |
         'metaget' @ { parser->msg->op = mc_op_metaget; } ;

arithmetic_op = 'incr' @ { parser->msg->op = mc_op_incr; } |
                'decr' @ { parser->msg->op = mc_op_decr; };

version_op = 'version' @ { parser->msg->op = mc_op_version; };

quit_op = 'quit' @ { parser->msg->op = mc_op_quit;  };

stats_op = 'stats' @ { parser->msg->op = mc_op_stats; };

exec_op = ('exec' | 'admin') @ { parser->msg->op = mc_op_exec; };

shutdown_op = 'shutdown' @ { parser->msg->op = mc_op_shutdown; };

flush_all_op = 'flush_all' @ { parser->msg->op = mc_op_flushall; };
flush_regex_op = 'flush_regex' @ { parser->msg->op = mc_op_flushre; };

cas = 'cas' %{
  parser->msg->op = mc_op_cas;
} ' '+ key ' '+ flags ' '+ exptime ' '+ value_bytes ' '+ casid ' '* nl
  @rx_data fin;

store = (set_op ' '+ key | leaseset ' '+ key ' '+ lease_token) ' '+ flags ' '+ exptime ' '+ value_bytes (' '+ noreply)? ' '* nl
  @rx_data fin;

get = get_op (' '+ key %get_req_is_ready)+ ' '* fin >{
  parser->msg->op = mc_op_end;
};

arithmetic = arithmetic_op ' '+ key ' '+ delta (' '+ noreply)? ' '* fin;

delete = delete_op ' '+ key (' '+ exptime)? (' '+ noreply)? ' '* fin;

quit = quit_op ' '* fin;

version = version_op ' '* fin;

stats = stats_op (' '+ multi_token)? ' '* fin;

exec = exec_op ' '+ multi_token ' '* fin;

shutdown = shutdown_op (' '+ number)? ' '* fin;

flush_all = flush_all_op (' '+ number)? ' '* fin;

flush_regex = flush_regex_op ' ' key ' '* fin;

request := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec;

request_reply := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec |
  get_reply | lease_get | metaget_reply |
  reply_set | reply_add | reply_replace | reply_lease_set |
  reply_append | reply_prepend | reply_cas | delete_reply |
  arithmetic_reply |
  stats_reply | ok | version_reply | error;

write data;
}%%

int get_reply_ragel_start_state() {
  return ascii_en_reply;
}

int get_request_ragel_start_state() {
  return ascii_en_request;
}

int get_request_reply_ragel_start_state() {
  return ascii_en_request_reply;
}

/** ascii reply parsing
    This is the real workhorse. It consumes buf, resuming the partial reply if
    one exists. As each reply is fully received, we call mcc_req_complete().
    If there is a parse error or remote error, we handle that as well. If
    there is a parse error or unrecoverable remote error, we call
    mcc_on_down() and the rest of the buffer is ignored.
    @return 0 on error */
int _on_ascii_rx(mc_parser_t* parser, char* buf, size_t nbuf) {
  FBI_ASSERT(parser);
  FBI_ASSERT(buf);
  dbg_fentry("parser=%p nbuf=%lu", parser, nbuf);

  int success = 0;

  // ragel variables
  int cs = ascii_error; // current state
  char* p = buf;
  char* pe = buf + nbuf;
  char* eof = NULL;
  %% write init nocs;

  char* ts = NULL; // token start
  size_t tlen = 0; // token length

  int nmsgs = 0;

  /* each iteration of this loop processes one msg, or in the case of value
     msgs the header of the msg, the body of the msg, or the tail of
     the msg. If we reach the end of buf before the end of a msg (whether
     in data or header/tail), we have a partial read and will resume when the
     next packet arrives. */
  while (p < pe) {
    /* sanity check: if memcached has a bug, or we somehow lost a msg, log
       the whatever's left and disconnect */
    if (parser->parser_state == parser_idle) {
      dbg_error("Unexpected msg '%s'", p);
      parser->parse_error(parser->context, parser->error);
      break;
    }

    FBI_ASSERT(parser->parser_state == parser_msg_header ||
           parser->parser_state == parser_body ||
           parser->parser_state == parser_partial);

    /* prepare to resume. If we're not in the middle of a msg, these will
       not be used */
    cs = parser->ragel_state;

    if (parser->parser_state == parser_msg_header) {
      if (parser->msg != NULL) {
        dbg_error("parser->msg already exists at start of new message. "
                  "Probable memory leak.");
      }

      // new msg
      parser->msg = mc_msg_new(0);
      if (parser->msg == NULL) {
        parser->error = parser_out_of_memory;
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        success = 0;
        goto epilogue;
      }

      cs = parser->ragel_start_state;
      parser->parser_state = parser_partial;
    }
    FBI_ASSERT(parser->msg);

    dbg_low("resid=%lu msg=%p parser_state=%s cs=%d",
            pe - p, parser->msg,
            parser_state_to_string(parser->parser_state), cs);

    if (parser->parser_state == parser_body) {
      FBI_ASSERT(pe >= p);
      size_t n = MIN(parser->resid, (size_t)(pe - p));
      memcpy(&parser->msg->value.str[parser->off], p, n);
      p += n;
      parser->off += n;
      FBI_ASSERT(parser->resid >= n);
      parser->resid -= n;
      if (parser->resid == 0) {
        FBI_ASSERT(parser->off == parser->msg->value.len);
        parser->parser_state = parser_partial;
      }
    } else {
      if (parser->partial_token) {
        // we're in the middle of a token
        ts = p;
      } else {
        ts = NULL;
      }

      /* this is where ragel emits the state machine code */
      %% write exec;

      /* if msg == NULL, we finished the msg */
      if (parser->msg) {
        parser->ragel_state = cs;
      } else {
        /* sentinel values */
        parser->ragel_state = ascii_error;
      }

      if (ts && parser->partial_token) {
        // we're in the middle of a token, stuff it in our cheeks
        int n = p - ts;
        if(mc_parser_ensure_tbuf(parser, n) == 0) {
          strncpy(parser->te, ts, n);
          parser->te += n;
        } else {
          if (parser->in_key) {
            parser->bad_key = 1;
          } else {
            // fake fgoto error here
            parser->error = parser_malformed_request;
            cs = ascii_error;
          }
        }
      }

      if (cs == ascii_error) {
        if (parser->msg != NULL) {
          mc_msg_decref(parser->msg);
          parser->msg = NULL;
        }
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        goto epilogue;
      }
    }

    /* We may have adjusted pe and eof to simulate EOF at the end of a msg.
       Restore them for continued parsing of additional msgs. */
    pe = buf + nbuf;
    eof = NULL;
  }

  success = 1;

epilogue:
  dbg_fexit("%d msgs finished. parser_state=%s cs=%d",
            nmsgs, parser_state_to_string(parser->parser_state), cs);
  return success;
}

%%{
# vim: ft=ragel
}%%

/* -*- mode: c; -*- */
/* ascii_client.c is built from ascii_client.rl using ragel */

#include <ctype.h>

#include "mcrouter/lib/fbi/debug.h"
#include "mcrouter/lib/mc/_protocol.h"
#include "mcrouter/lib/mc/parser.h"
#include "mcrouter/lib/mc/msg.h"
#include "mcrouter/lib/mc/util.h"

// In libmc.client, only read the first N stats for simplicity of
// implementation. In the future it might make sense to allow an
// arbitrary number of stats, but for now that makes the code far
// easier
#define MAX_NUM_STATS 20

// Are we in the middle of a key being skipped?
#define skipping_key(parser) ((parser)->in_skipped_key &&       \
                              !(parser)->record_skip_key)

%%{

# input file for ragel (http://www.complang.org/ragel/)

# this machine name, which must come first, is used as a prefix for variables
machine ascii;

# named actions. All the actions (named and unnamed) are executed in the
# context of _on_ascii_rx() where "%% write exec;" is called.
action start_token {
  if (!skipping_key(parser)) {
    parser->partial_token = 1;
    parser->te = parser->tbuf;
    ts = p;
  }
}
action finish_token {
  if (parser->partial_token && !skipping_key(parser)) {
    FBI_ASSERT(ts);
    parser->partial_token = 0;
    if (parser->te && parser->te > parser->tbuf) {
      /* we have something stuffed in our cheeks,
         let's put them together and update ts. */
      FBI_ASSERT(ts <= p);
      size_t n = p - ts;
      if(mc_parser_ensure_tbuf(parser, n) != 0) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      } else {
        strncpy(parser->te, ts, n);
        parser->te += n;
        *parser->te = '\0';
        tlen = parser->te - parser->tbuf;
        ts = parser->tbuf;
      }
    } else {
      tlen = p - ts;
      if (tlen >= MC_TOKEN_MAX_LEN) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      }
    }
  }
}

action record_key {
  if (!skipping_key(parser)) {
    if (tlen > MC_KEY_MAX_LEN_ASCII) {
      parser->bad_key = 1;
    }
    if (parser->bad_key) {
      parser->msg->result = mc_res_bad_key;
    } else {
      if (mc_msg_grow(&parser->msg, tlen + 1,
                      (void**) &parser->msg->key.str) != 0) {
        parser->error = parser_out_of_memory;
        fgoto error_handling;
      }
      parser->msg->key.len = tlen;
      strncpy(parser->msg->key.str, ts, tlen);
      FBI_ASSERT(!memcmp(parser->msg->key.str, ts, tlen));
      parser->msg->key.str[tlen] = '\0';
    }
  }
  parser->in_skipped_key = 0;
  parser->in_key = 0;
  parser->bad_key = 0;
}

action fin {
  FBI_ASSERT(parser->msg != NULL);

  // end of msg, fake eof
  pe = eof = p + 1;
  parser->parser_state = parser_msg_header;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = NULL;
  parser->msg_ready(parser->context, 0, msg);
  nmsgs++;

  mc_parser_cleanup_tbuf(parser);
}

nl = '\r'? '\n';
fin = nl @fin;

# server-reported errors. We should only ever see SERVER_ERROR. If we see
# CLIENT_ERROR then libmc probably didn't do enough error checking before
# sending to memcache in the first place. ERROR is absolutely a bug in libmc.
error_handling := '' >{
  if (parser->msg != NULL) {
    mc_msg_decref(parser->msg);
  }
  parser->msg = NULL;
  parser->te = NULL;
  parser->in_key = 0;
  parser->bad_key = 0;
  fbreak;
};

error_code = ' ' digit+ ' '
  @{
    parser->msg->err_code = strtoll(ts, NULL, 10);
  };

action capture_error_value {
  // Since the token we are parsing includes the \r\n, we need
  // to invoke this code upon entering the DFA's final state.
  // Only the '\r' is stored in the token buffer, so we only
  // need to remove one character.
  tlen--;
  if (mc_msg_grow(&parser->msg, tlen + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.len = tlen;
  memcpy(parser->msg->value.str, ts, tlen);
  parser->msg->value.str[tlen] = '\0';
}

# server error apparently needs to be able to read binary data to support
# TAO, which uses SERVER_ERROR to send the fbid of objects back to the
# client during set requests.  There is no size field, so we need to
# keep parsing the response string until we hit \r\n.  Server error
# 307 is returned by a server that is overloaded or is shutting down;
# it is treated specially so that it can cross the application/transport
# boundary.  (mc_res_busy is an artifact of our use of TCP/IP, of our
# desired to keep the connection open while not accepting requests.
# If there was a reliable request-based protocol it would be less
# necessary.)
server_error = 'SERVER_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    if (parser->msg->err_code == SERVER_ERROR_BUSY) {
      parser->msg->result = mc_res_busy;
    } else {
      parser->msg->result = mc_res_remote_error;
    }
  }
  @fin;

# client error message may contain useful stuff, so we also want to save
# it for the caller
client_error = 'CLIENT_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    parser->msg->result = mc_res_client_error;
  }
  @fin;

command_error = 'ERROR'
  @{
    parser->msg->result = mc_res_local_error;
    parser->msg->value.str = "unexpected ERROR reply";
    parser->msg->value.len = strlen(parser->msg->value.str);
    dbg_error("%s", parser->msg->value.str);
    FBI_ASSERT(0);
  } fin;

error = server_error | client_error | command_error;

# Get request is ready to be passed to the caller
action get_req_is_ready {
  FBI_ASSERT(parser->msg != NULL);

  mc_op_t op = parser->msg->op;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = mc_msg_new(0);
  parser->msg_ready(parser->context, 0, msg);
  if (parser->msg == NULL) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }

  parser->msg->op = op;
}

# building blocks
stored = 'STORED' @{ parser->msg->result = mc_res_stored; };
stale_stored = 'STALE_STORED' @{
  parser->msg->result = mc_res_stalestored;
} fin;
not_stored = 'NOT_STORED' @{ parser->msg->result = mc_res_notstored; };
exists = 'EXISTS' @{ parser->msg->result = mc_res_exists; };
not_found = 'NOT_FOUND' @{ parser->msg->result = mc_res_notfound; };
deleted = 'DELETED' @{ parser->msg->result = mc_res_deleted; };

storage_reply = stored | stale_stored | not_stored | exists | not_found | deleted;
storage = storage_reply fin;
arithmetic_reply = (not_found |
  digit+ >start_token %finish_token %{
    parser->msg->delta = strtoll(ts, NULL, 10);
    parser->msg->result = mc_res_stored;
  } ' '* fin);

key_skip = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 1; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

key = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 0; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

# This is used for stats and exec commands which can have keys with spaces.
multi_token = ' '* (print+ -- ( '\r' | '\n' ))
  >start_token %finish_token %{
  while (tlen > 0 && isspace(ts[tlen-1])) {
    tlen--;
  }
  if (mc_msg_grow(&parser->msg, tlen+1, (void**) &parser->msg->key.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->key.len = tlen;
  strncpy(parser->msg->key.str, ts, tlen);
  parser->msg->key.str[tlen] = '\0';
};

lease_token = ('-'? digit+) >start_token %finish_token %{
  parser->msg->lease_id = strtol(ts, NULL, 10);
};
casid = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoll(ts, NULL, 10);
};
flags = digit+ >start_token %finish_token %{
  parser->msg->flags = strtoll(ts, NULL, 10);
};
exptime = digit+ >start_token %finish_token %{
  parser->msg->exptime = strtol(ts, NULL, 10);
};
delta = digit+ >start_token %finish_token %{
  parser->msg->delta = strtoll(ts, NULL, 10);
};
number = digit+ >start_token %finish_token %{
  parser->msg->number = strtoll(ts, NULL, 10);
};
noreply = 'noreply' %{
  parser->msg->noreply = true;
};

value_bytes = digit+ >start_token %finish_token %{
  /* prepare for receiving data */
  size_t len = parser->resid = strtol(ts, NULL, 10);
  parser->off = 0;

  if (mc_msg_grow(&parser->msg, len + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.str[len] = '\0';
  parser->msg->value.len = len;
};
cas_unique = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoull(ts, NULL, 10);
};
dec_octet = digit{1,3};
ipv4_addr = dec_octet "." dec_octet "." dec_octet "." dec_octet;
ipv6_addr = (":" | xdigit)+ ipv4_addr?;
ip_addr = (ipv4_addr | ipv6_addr) >start_token %finish_token %{
  ts[tlen]='\0';
  parser->msg->ipv = 0;
  if (strchr(ts, ':') == NULL) {
    if (inet_pton(AF_INET, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 4;
  } else {
    if (inet_pton(AF_INET6, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 6;
  }
};
age_unknown = 'unknown' %{ parser->msg->number = -1;};
action rx_data {
  /* the binary data of a specified isn't really regular, so we drop out of
     the state machine and read it, then come back in to finish the
     "\r\nEND\r\n". */
  parser->parser_state = parser_body;
  fbreak;
}

# replies

VALUE = 'VALUE' % { parser->msg->result = mc_res_found; };

hit = VALUE ' '+ key_skip ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
  @rx_data nl;

get_reply = (hit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
lease_get = ((hit |
  ('LVALUE' ' '+ key_skip ' '+ lease_token ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
   @rx_data nl
   @{ parser->msg->result = mc_res_notfound; }))
  'END' fin);
META = 'META' % {parser->msg->result = mc_res_found;};
mhit = META ' '+ key_skip ' '+ 'age:' ' '* (number | age_unknown) ';' ' '*
  'exptime:' ' '* exptime ';' ' '* 'from:' ' '* (ip_addr|'unknown') ';' ' '*
  'is_transient:' ' '* flags ' '* nl;
metaget_reply = (mhit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
reply_set = storage;
reply_add = storage;
reply_replace = storage;
reply_lease_set = storage;
reply_append = storage;
reply_prepend = storage;
reply_cas = storage;
delete_reply = deleted | not_found;

stat_name = (any+ -- (cntrl | space)) >start_token %finish_token
%{
  if (parser->msg->stats == NULL) {
    FBI_ASSERT(parser->msg->number == 0);
    size_t stats_size = sizeof(nstring_t) * 2 * MAX_NUM_STATS;
    if (mc_msg_grow(&parser->msg, stats_size, (void**) &parser->msg->stats)) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    memset(parser->msg->stats, 0, stats_size);
  }

  if (parser->msg->number < MAX_NUM_STATS) {
    parser->msg->number++;

    int idx = (parser->msg->number * 2) - 2;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stat_value = (any++ -- cntrl) > start_token %finish_token
%{
  if (parser->msg->number <= MAX_NUM_STATS) {
    int idx = (parser->msg->number * 2) - 1;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }

    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stats_reply = (('STAT ' @{ parser->msg->result = mc_res_ok; }
                stat_name ' '+ stat_value nl)* 'END' fin);
ok = ('OK' @{ parser->msg->result = mc_res_ok; } fin);
version_reply = ('VERSION ' (any* -- ('\r' | '\n')) >start_token %finish_token
  %{
    // use new_msg because msg stays live if realloc fails and can leak
    mc_msg_t *new_msg = mc_msg_realloc(parser->msg,
                                       parser->msg->_extra_size + tlen + 1);
    if (new_msg == NULL) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg = new_msg;
    parser->msg->result = mc_res_ok;
    char* version = parser->msg->value.str = (char *) &parser->msg[1];
    parser->msg->value.len = tlen;
    strncpy(version, ts, tlen);
    version[tlen] = '\0';
  }
  fin);

reply := get_reply | lease_get | metaget_reply |
        reply_set | reply_add | reply_replace | reply_lease_set |
        reply_append | reply_prepend | reply_cas | delete_reply |
        arithmetic_reply |
        stats_reply | ok | version_reply | error;

# requests

set = 'set' @ { parser->msg->op = mc_op_set; };
add = 'add' @ { parser->msg->op = mc_op_add; };
replace = 'replace' @ { parser->msg->op = mc_op_replace; };
leaseset = 'lease-set' @ { parser->msg->op = mc_op_lease_set; };
append = 'append' @ { parser->msg->op = mc_op_append; };
prepend = 'prepend' @ { parser->msg->op = mc_op_prepend; };

set_op = set | add | replace | append | prepend;

delete_op = 'delete' @ { parser->msg->op = mc_op_delete; };

get_op = 'get' @ { parser->msg->op = mc_op_get; } |
         'gets' @ {parser->msg->op = mc_op_gets; } |
         'lease-get' @ { parser->msg->op = mc_op_lease_get; } |
         'metaget' @ { parser->msg->op = mc_op_metaget; } ;

arithmetic_op = 'incr' @ { parser->msg->op = mc_op_incr; } |
                'decr' @ { parser->msg->op = mc_op_decr; };

version_op = 'version' @ { parser->msg->op = mc_op_version; };

quit_op = 'quit' @ { parser->msg->op = mc_op_quit;  };

stats_op = 'stats' @ { parser->msg->op = mc_op_stats; };

exec_op = ('exec' | 'admin') @ { parser->msg->op = mc_op_exec; };

shutdown_op = 'shutdown' @ { parser->msg->op = mc_op_shutdown; };

flush_all_op = 'flush_all' @ { parser->msg->op = mc_op_flushall; };
flush_regex_op = 'flush_regex' @ { parser->msg->op = mc_op_flushre; };

cas = 'cas' %{
  parser->msg->op = mc_op_cas;
} ' '+ key ' '+ flags ' '+ exptime ' '+ value_bytes ' '+ casid nl
  @rx_data fin;

store = (set_op ' '+ key | leaseset ' '+ key ' '+ lease_token) ' '+ flags ' '+ exptime ' '+ value_bytes (' '+ noreply)? nl
  @rx_data fin;

get = get_op (' '+ key %get_req_is_ready)+  fin >{
  parser->msg->op = mc_op_end;
};

arithmetic = arithmetic_op ' '+ key ' '+ delta (' '+ noreply)? fin;

delete = delete_op ' '+ key (' '+ exptime)? (' '+ noreply)? fin;

quit = quit_op fin;

version = version_op fin;

stats = stats_op (' '+ multi_token)? fin;

exec = exec_op ' '+ multi_token fin;

shutdown = shutdown_op (' '+ number)? fin;

flush_all = flush_all_op (' '+ number)? fin;

flush_regex = flush_regex_op ' ' key fin;

request := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec;

request_reply := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec |
  get_reply | lease_get | metaget_reply |
  reply_set | reply_add | reply_replace | reply_lease_set |
  reply_append | reply_prepend | reply_cas | delete_reply |
  arithmetic_reply |
  stats_reply | ok | version_reply | error;

write data;
}%%

int get_reply_ragel_start_state() {
  return ascii_en_reply;
}

int get_request_ragel_start_state() {
  return ascii_en_request;
}

int get_request_reply_ragel_start_state() {
  return ascii_en_request_reply;
}

/** ascii reply parsing
    This is the real workhorse. It consumes buf, resuming the partial reply if
    one exists. As each reply is fully received, we call mcc_req_complete().
    If there is a parse error or remote error, we handle that as well. If
    there is a parse error or unrecoverable remote error, we call
    mcc_on_down() and the rest of the buffer is ignored.
    @return 0 on error */
int _on_ascii_rx(mc_parser_t* parser, char* buf, size_t nbuf) {
  FBI_ASSERT(parser);
  FBI_ASSERT(buf);
  dbg_fentry("parser=%p nbuf=%lu", parser, nbuf);

  int success = 0;

  // ragel variables
  int cs = ascii_error; // current state
  char* p = buf;
  char* pe = buf + nbuf;
  char* eof = NULL;
  %% write init nocs;

  char* ts = NULL; // token start
  size_t tlen = 0; // token length

  int nmsgs = 0;

  /* each iteration of this loop processes one msg, or in the case of value
     msgs the header of the msg, the body of the msg, or the tail of
     the msg. If we reach the end of buf before the end of a msg (whether
     in data or header/tail), we have a partial read and will resume when the
     next packet arrives. */
  while (p < pe) {
    /* sanity check: if memcached has a bug, or we somehow lost a msg, log
       the whatever's left and disconnect */
    if (parser->parser_state == parser_idle) {
      dbg_error("Unexpected msg '%s'", p);
      parser->parse_error(parser->context, parser->error);
      break;
    }

    FBI_ASSERT(parser->parser_state == parser_msg_header ||
           parser->parser_state == parser_body ||
           parser->parser_state == parser_partial);

    /* prepare to resume. If we're not in the middle of a msg, these will
       not be used */
    cs = parser->ragel_state;

    if (parser->parser_state == parser_msg_header) {
      if (parser->msg != NULL) {
        dbg_error("parser->msg already exists at start of new message. "
                  "Probable memory leak.");
      }

      // new msg
      parser->msg = mc_msg_new(0);
      if (parser->msg == NULL) {
        parser->error = parser_out_of_memory;
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        success = 0;
        goto epilogue;
      }

      cs = parser->ragel_start_state;
      parser->parser_state = parser_partial;
    }
    FBI_ASSERT(parser->msg);

    dbg_low("resid=%lu msg=%p parser_state=%s cs=%d",
            pe - p, parser->msg,
            parser_state_to_string(parser->parser_state), cs);

    if (parser->parser_state == parser_body) {
      FBI_ASSERT(pe >= p);
      size_t n = MIN(parser->resid, (size_t)(pe - p));
      memcpy(&parser->msg->value.str[parser->off], p, n);
      p += n;
      parser->off += n;
      FBI_ASSERT(parser->resid >= n);
      parser->resid -= n;
      if (parser->resid == 0) {
        FBI_ASSERT(parser->off == parser->msg->value.len);
        parser->parser_state = parser_partial;
      }
    } else {
      if (parser->partial_token) {
        // we're in the middle of a token
        ts = p;
      } else {
        ts = NULL;
      }

      /* this is where ragel emits the state machine code */
      %% write exec;

      /* if msg == NULL, we finished the msg */
      if (parser->msg) {
        parser->ragel_state = cs;
      } else {
        /* sentinel values */
        parser->ragel_state = ascii_error;
      }

      if (ts && parser->partial_token) {
        // we're in the middle of a token, stuff it in our cheeks
        int n = p - ts;
        if(mc_parser_ensure_tbuf(parser, n) == 0) {
          strncpy(parser->te, ts, n);
          parser->te += n;
        } else {
          if (parser->in_key) {
            parser->bad_key = 1;
          } else {
            // fake fgoto error here
            parser->error = parser_malformed_request;
            cs = ascii_error;
          }
        }
      }

      if (cs == ascii_error) {
        if (parser->msg != NULL) {
          mc_msg_decref(parser->msg);
          parser->msg = NULL;
        }
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        goto epilogue;
      }
    }

    /* We may have adjusted pe and eof to simulate EOF at the end of a msg.
       Restore them for continued parsing of additional msgs. */
    pe = buf + nbuf;
    eof = NULL;
  }

  success = 1;

epilogue:
  dbg_fexit("%d msgs finished. parser_state=%s cs=%d",
            nmsgs, parser_state_to_string(parser->parser_state), cs);
  return success;
}

%%{
# vim: ft=ragel
}%%