teach zone.rl to parse SPF records

teach zone.rl to parse SPF records

/* ==========================================================================
 * zone.rl - RFC 1035 Master File Parser
 * --------------------------------------------------------------------------
 * Copyright (c) 2010  William Ahern
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the
 * following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
 * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 * ==========================================================================
 */
#include <stddef.h>	/* NULL */
#include <stdlib.h>	/* malloc(3) free(3) */
#include <stdio.h>	/* fopen(3) fclose(3) fread(3) fputc(3) */

#include <string.h>	/* memset(3) memmove(3) */

#include <ctype.h>	/* isspace(3) isgraph(3) isdigit(3) */

#include <assert.h>	/* assert(3) */

#include <errno.h>	/* errno */

#include <arpa/inet.h>	/* inet_pton(3) */

#include "dns.h"
#include "zone.h"

#ifndef lengthof
#define lengthof(a) (sizeof (a) / sizeof (a)[0])
#endif

#ifndef endof
#define endof(a) (&(a)[lengthof(a)])
#endif

#ifndef MIN
#define MIN(a, b) (((a) < (b))? (a) : (b))
#endif

#define SAY_(fmt, ...) \
	do { fprintf(stderr, fmt "%.1s", __func__, __LINE__, __VA_ARGS__); } while (0)
#define SAY(...) SAY_(">>>> (%s:%d) " __VA_ARGS__, "\n")
#define HAI SAY("HAI")

/*
 * P R E - P R O C E S S I N G  R O U T I N E S
 *
 * Outputs an encoded normal form of the RFC1035 master file syntax.
 *
 * o Linear spaces and tabs trimmed.
 *
 * o Multi-line groups are folded, with grouping parentheses replaced by
 *   spaces.
 *
 * o Comments are discarded.
 *
 * o Quoted and escaped whitespace characters are coded as a short with the
 *   T_LIT bit set.
 *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#define T_LIT 0x0100

struct zonepp {
	int escaped, quoted, grouped, comment, lastc;
	struct { int v, n; } octet;
}; /* struct zonepp */

#define islwsp(ch) ((ch) == ' ' || (ch) == '\t')

static size_t fold(short *dst, size_t lim, unsigned char **src, size_t *len, struct zonepp *state) {
	short *p, *pe;
	int ch;

	if (!state->lastc)
		state->lastc = '\n';

	p  = dst;
	pe = p + lim;

	while (p < pe && *len) {
		ch = **src;

		if (state->escaped) {
			if (state->octet.n && (state->octet.n >= 3 || !isdigit(ch))) {
				state->escaped = 0;

				ch = state->octet.v;
				state->octet.v = 0;
				state->octet.n = 0;

				if (isspace(ch))
					ch |= T_LIT;

				*p++ = ch;

				continue; /* don't skip current char */
			} else if (isdigit(ch)) {
				state->octet.v *= 10;
				state->octet.v += ch - '0';
				state->octet.n++;

				goto skip;
			} else {
				state->escaped = 0;

				if (isspace(ch))
					ch |= T_LIT;

				goto copy;
			}
		} else if (state->comment) {
			if (ch != '\n')
				goto skip;

			state->comment = 0;
		}

test:
		switch (ch) {
		case '\\':
			state->escaped = 1;

			goto skip;
		case ';':
			if (state->quoted)
				break;

			state->comment = 1;

			goto skip;
		case '"':
			state->quoted = !state->quoted;

			goto skip;
		case '(':
			if (state->quoted || state->grouped)
				break;

			state->grouped = 1;

			ch = ' ';

			goto test;
		case ')':
			if (state->quoted || !state->grouped)
				break;

			state->grouped = 0;

			ch = ' ';

			goto test;
		case '\n':
			if (state->quoted) {
				break;
			} else if (state->lastc == '\n') {
				goto skip;
			} else if (state->grouped) {
				ch = ' ';

				goto test;
			}

			break;
		case '\t':
			ch = ' ';

			goto test;
		case ' ':
			if (state->quoted)
				break;
			else if (islwsp(state->lastc))
				goto skip;

			break;
		} /* switch() */
copy:
		if (state->quoted && isspace((0xff & ch)))
			ch |= T_LIT;

		state->lastc = *p = ch;

		++p;
skip:
		++*src;
		--*len;
	} /* while() */

	return p - dst;
} /* fold() */

struct tokens {
	short base[4096];
	unsigned count;
}; /* struct tokens */

static unsigned tok_eol(struct tokens *b) {
	unsigned i;

	for (i = 0; i < b->count; i++) {
		if (b->base[i] == '\n')
			return i + 1;
	}

	return 0;
} /* tok_eol() */

static void tok_discard(struct tokens *b, unsigned count) {
	if (count) {
		memmove(b->base, &b->base[count], 2 * (b->count - count));
		b->count = b->count - count;
	}
} /* tok_discard() */

struct zonefile {
	unsigned ttl;
	char origin[DNS_D_MAXNAME + 1];
	char lastrr[DNS_D_MAXNAME + 1];

	struct dns_soa soa;

	struct tokens toks;
	struct zonepp pp;
}; /* struct zonefile */

void zone_init(struct zonefile *P, const char *origin, unsigned ttl) {
	memset(P, 0, sizeof *P);

	origin = (origin)? origin : ".";
	dns_strlcpy(P->origin, origin, sizeof P->origin);
	dns_strlcpy(P->lastrr, origin, sizeof P->lastrr);

	P->ttl = (ttl)? ttl : 3600;
} /* zone_init() */

void zonerr_init(struct zonerr *rr, struct zonefile *P) {
	memset(rr, 0, sizeof *rr);
	rr->class = DNS_C_IN;
	rr->ttl   = P->ttl;
	dns_any_init(&rr->data, sizeof rr->data);
} /* zonerr_init() */

%%{
	machine file_grammar;
	alphtype short;

	action oops {
		fprintf(stderr, "OOPS: ");
		do {
			unsigned i, at = p - P->toks.base;
			for (i = 0; i < span; i++) {
				if (i == at) fputc('[', stderr);
				fputc((0xff & P->toks.base[i]), stderr);
				if (i == at) fputc(']', stderr);
			}
		} while(0);
		goto next;
	}

	action str_init { sp = str; }
	action str_copy { if (sp < endof(str)) *sp++ = fc; lc = fc; }
	action str_end { if (sp >= endof(str)) sp--; *sp = '\0'; }

	string = (any - space)+ >str_init $str_copy %str_end;

	number = digit+ >{ n = 0; } ${ n *= 10; n += fc - '0'; };

	ttl_unit =
		("w"i @{ ttl *= 604800; }) |
		("d"i @{ ttl *= 86400; }) |
		("h"i @{ ttl *= 3600; }) |
		("m"i @{ ttl *= 60; }) |
		("s"i);
	ttl = number %{ ttl = n; } ttl_unit? space*;

	action dom_end {
		 if (lc != '.') {
			if (P->origin[0] != '.')
				dns_strlcat(str, ".", sizeof str);
			dns_strlcat(str, P->origin, sizeof str);
		}
	}

	domain = string %dom_end;

	SOA_type    = "SOA"i %{ rr->type = DNS_T_SOA; };
	SOA_mname   = domain %{ dns_strlcpy(rr->data.soa.mname, str, sizeof rr->data.soa.mname); };
	SOA_rname   = string %{ dns_strlcpy(rr->data.soa.rname, str, sizeof rr->data.soa.rname); };
	SOA_serial  = number %{ rr->data.soa.serial = n; };
	SOA_refresh = ttl %{ rr->data.soa.refresh = ttl; };
	SOA_retry   = ttl %{ rr->data.soa.retry = ttl; };
	SOA_expire  = ttl %{ rr->data.soa.expire = ttl; };
	SOA_minimum = ttl %{ rr->data.soa.minimum = ttl; };
	SOA = SOA_type space+ SOA_mname space+ SOA_rname space+ SOA_serial space+ SOA_refresh space+ SOA_retry space+ SOA_expire space+ SOA_minimum space*;

	NS_type = "NS"i %{ rr->type = DNS_T_NS; };
	NS_host = domain %{ dns_strlcpy(rr->data.ns.host, str, sizeof rr->data.ns.host); };
	NS = NS_type space+ NS_host space*;

	MX_type = "MX"i %{ rr->type = DNS_T_MX; };
	MX_pref = number %{ rr->data.mx.preference = n; };
	MX_host = domain %{ dns_strlcpy(rr->data.mx.host, str, sizeof rr->data.mx.host); };
	MX = MX_type space+ MX_pref space+ MX_host space*;

	CNAME_type = "CNAME"i %{ rr->type = DNS_T_CNAME; };
	CNAME_host = domain %{ dns_strlcpy(rr->data.cname.host, str, sizeof rr->data.cname.host); };
	CNAME = CNAME_type space+ CNAME_host space*;

	SRV_type   = "SRV"i %{ rr->type = DNS_T_SRV; };
	SRV_pri    = number %{ rr->data.srv.priority = n; };
	SRV_weight = number %{ rr->data.srv.weight = n; };
	SRV_port   = number %{ rr->data.srv.port = n; };
	SRV_target = domain %{ dns_strlcpy(rr->data.srv.target, str, sizeof rr->data.srv.target); };
	SRV = SRV_type space+ SRV_pri space+ SRV_weight space+ SRV_port space+ SRV_target space*;

	# FIXME: Support multiple segments.
	TXT_type = "TXT"i %{ rr->type = DNS_T_TXT; };
	TXT_data = string %{ rr->data.txt.len = MIN(rr->data.txt.size - 1, sp - str); memcpy(rr->data.txt.data, str, rr->data.txt.len); };
	TXT = TXT_type space+ TXT_data space*;

	SPF_type = "SPF"i %{ rr->type = DNS_T_SPF; };
	SPF = SPF_type space+ TXT_data space*;

	action AAAA_pton {
		if (1 != inet_pton(AF_INET6, str, &rr->data.aaaa.addr)) {
			SAY("invalid AAAA address: %s", str);
			goto next;
		}
	}

	AAAA_type   = "AAAA"i %{ rr->type = DNS_T_AAAA; };
	AAAA_digits = (xdigit | "." | ":")+ >str_init $str_copy %str_end;
	AAAA_addr   = AAAA_digits %AAAA_pton;
	AAAA = AAAA_type space+ AAAA_addr space*;

	A_type  = "A"i %{ rr->type = DNS_T_A; };
	A_octet = digit{1,3} >{ n = 0; } ${ n *= 10; n += fc - '0'; } %{ rr->data.a.addr.s_addr <<= 8; rr->data.a.addr.s_addr |= (0xff & n); };
	A_addr  = (A_octet "." A_octet "." A_octet "." A_octet) %{ rr->data.a.addr.s_addr = htonl(rr->data.a.addr.s_addr); };
	A = A_type space+ A_addr space*;

	PTR_type  = "PTR"i %{ rr->type = DNS_T_PTR; };
	PTR_cname = domain %{ dns_strlcpy(rr->data.ptr.host, str, sizeof rr->data.ptr.host); };
	PTR = PTR_type space+ PTR_cname space*;

	rrdata = (SOA | NS | MX | CNAME | SRV | TXT | SPF | AAAA | A | PTR) space*;

	rrname_blank  = " " %{ dns_strlcpy(rr->name, P->lastrr, sizeof rr->name); } " "*;
	rrname_origin = "@ " %{ dns_strlcpy(rr->name, P->origin, sizeof rr->name); } " "*;
	rrname_plain  = domain %{ if (!rr->name[0]) dns_strlcpy(rr->name, str, sizeof rr->name); } " "+;

	rrname = rrname_blank | rrname_origin | rrname_plain;

	rrclass = "IN " %{ rr->class = DNS_C_IN; };

	rrttl = ttl %{ rr->ttl = ttl; } space+;

	zonerr = rrname rrttl? rrclass? rrttl? rrdata;

	ORIGIN = "$ORIGIN " domain %{ dns_strlcpy(P->origin, str, sizeof P->origin); goto next; } space*;

	TTL    = "$TTL " ttl %{ P->ttl = ttl; goto next; } space*;

	nothing = space* %{ goto next; };

	main := (ORIGIN | TTL | zonerr | nothing) $!oops;
}%%

struct zonerr *zone_getrr(struct zonerr *rr, struct dns_soa **soa, struct zonefile *P) {
	%% write data;
	short *p, *pe, *eof;
	int cs;
	char str[1024], *sp, lc;
	unsigned span, ttl, n;

	span = 0;
next:
	tok_discard(&P->toks, span);

	if (!(span = tok_eol(&P->toks)))
		return NULL;

	sp  = str;
	lc  = 0;
	ttl = 0;
	n   = 0;

	zonerr_init(rr, P);

	%% write init;

	p   = P->toks.base;
	pe  = p + span;
	eof = pe;

	%% write exec;

	tok_discard(&P->toks, span);

	dns_strlcpy(P->lastrr, rr->name, sizeof P->lastrr);

	if (rr->type == DNS_T_SOA)
		P->soa = rr->data.soa;

	*soa = &P->soa;

	return rr;
} /* zone_getrr() */

size_t zone_parsesome(struct zonefile *P, const void *src, size_t len) {
	unsigned char *p = (unsigned char *)src;
	size_t lim = lengthof(P->toks.base) - P->toks.count;

	P->toks.count += fold(&P->toks.base[P->toks.count], lim, &p, &len, &P->pp);

	return p - (unsigned char *)src;
} /* zone_parsesome() */

size_t zone_parsefile(struct zonefile *P, FILE *fp) {
	unsigned char buf[1024];
	size_t tlim, lim, len;

	tlim = lengthof(P->toks.base) - P->toks.count;
	lim  = MIN(lengthof(buf), tlim);

	if (!(len = fread(buf, 1, lim, fp)))
		return 0;

	assert(len == zone_parsesome(P, buf, len));

	return len;
} /* zone_parsefile() */

struct zonefile *zone_open(const char *origin, unsigned ttl, int *error) {
	struct zonefile *P;

	if (!(P = malloc(sizeof *P)))
		{ *error = errno; return NULL; }

	zone_init(P, origin, ttl);

	return P;
} /* zone_open() */

void zone_close(struct zonefile *P) {
	free(P);
} /* zone_close() */

#if ZONE_MAIN

#include <stdlib.h>	/* EXIT_FAILURE */

#include <unistd.h>	/* getopt(3) */

struct {
	const char *progname;
} MAIN;

static void parsefile(FILE *fp, const char *origin, unsigned ttl) {
	struct zonefile P;
	struct zonerr rr;
	struct dns_soa *soa;

	zone_init(&P, origin, ttl);

	while (zone_parsefile(&P, fp)) {
		while (zone_getrr(&rr, &soa, &P)) {
			char data[512];
			dns_any_print(data, sizeof data, &rr.data, rr.type);
			printf("%s %u IN %s %s\n", rr.name, rr.ttl, dns_strtype(rr.type), data);
		}
	}
} /* parsefile() */

static void foldfile(FILE *dst, FILE *src) {
	struct zonepp state;
	unsigned char in[16], *p;
	short out[16];
	size_t len, num, i;

	memset(&state, 0, sizeof state);

	while ((len = fread(in, 1, sizeof in, src))) {
		p = in;

		while ((num = fold(out, sizeof out, &p, &len, &state))) {
			for (i = 0; i < num; i++) {
				if ((out[i] & T_LIT)
				||  (!isgraph(0xff & out[i]) && !isspace(0xff & out[i]))) {
					fprintf(dst, "\\%.3d", (0xff & out[i]));
				} else {
					fputc((0xff & out[i]), dst);
				}
			}
		}
	}
} /* foldfile() */

static void usage(FILE *fp) {
	static const char *usage =
		" [OPTIONS] [COMMAND]\n"
		"  -o ORIGIN  Zone origin\n"
		"  -t TTL     Zone TTL\n"
		"  -V         Print version info\n"
		"  -h         Print this usage message\n"
		"\n"
		"  fold       Fold into simple normal form\n"
		"  parse      Parse zone file and recompose\n"
		"\n"
		"Report bugs to William Ahern <william@25thandClement.com>\n";

	fputs(MAIN.progname, fp);
	fputs(usage, fp);
	fflush(fp);
} /* usage() */

static unsigned parsettl(const char *opt) {
	unsigned ttl;
	char *end;

	ttl = strtoul(opt, &end, 10);

	switch (tolower((unsigned char)*end)) {
	case 'w':
		ttl *= 7;
	case 'd':
		ttl *= 24;
	case 'h':
		ttl *= 60;
	case 'm':
		ttl *= 60;
	case 's':
		ttl *= 1;
	case '\0':
		break;
	default:
		fprintf(stderr, "%s: ", MAIN.progname);

		for (; *opt; opt++) {
			if (opt == end)
				fprintf(stderr, "[%c]", *opt);
			else
				fputc(*opt, stderr);
		}
		fputs(": invalid TTL\n", stderr);

		exit(EXIT_FAILURE);
	}

	return ttl;
} /* parsettl() */

#define CMD_FOLD  1
#define CMD_PARSE 2

static int command(const char *arg) {
	const char *p, *pe, *eof;
	int cs;

	arg = (arg && *arg)? arg : "parse";

	p   = arg;
	pe  = p + strlen(arg);
	eof = pe;

	%%{
		machine command;

		action oops {
			fprintf(stderr, "%s: %s: invalid command\n", MAIN.progname, arg);
			usage(stderr);
			exit(EXIT_FAILURE);
		}

		fold = "fold" %{ return CMD_FOLD; };
		parse = "parse" %{ return CMD_PARSE; };

		main := (fold | parse) $!oops;

		write data;
		write init;
		write exec;
	}%%

	return 0;
} /* command() */

int main(int argc, char **argv) {
	extern int optind;
	extern char *optarg;
	int opt;
	const char *origin = ".";
	unsigned ttl = 3600;

	MAIN.progname = argv[0];

	while (-1 != (opt = getopt(argc, argv, "o:t:Vh"))) {
		switch (opt) {
		case 'o':
			origin = optarg;

			break;
		case 't':
			ttl = parsettl(optarg);

			break;
		case 'h':
			usage(stdout);

			return 0;
		default:
			usage(stderr);

			return EXIT_FAILURE;
		} /* switch() */
	} /* getopt() */

	argc -= optind;
	argv += optind;

	switch (command(argv[0])) {
	case CMD_FOLD:
		foldfile(stdout, stdin);
		break;
	case CMD_PARSE:
		parsefile(stdin, origin, ttl);
		break;
	}

	return 0;
} /* main() */

#endif /* ZONE_MAIN */

/* ==========================================================================
 * zone.rl - RFC 1035 Master File Parser
 * --------------------------------------------------------------------------
 * Copyright (c) 2010  William Ahern
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the
 * following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
 * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 * ==========================================================================
 */
#include <stddef.h>	/* NULL */
#include <stdlib.h>	/* malloc(3) free(3) */
#include <stdio.h>	/* fopen(3) fclose(3) fread(3) fputc(3) */

#include <string.h>	/* memset(3) memmove(3) */

#include <ctype.h>	/* isspace(3) isgraph(3) isdigit(3) */

#include <assert.h>	/* assert(3) */

#include <errno.h>	/* errno */

#include <arpa/inet.h>	/* inet_pton(3) */

#include "dns.h"
#include "zone.h"

#ifndef lengthof
#define lengthof(a) (sizeof (a) / sizeof (a)[0])
#endif

#ifndef endof
#define endof(a) (&(a)[lengthof(a)])
#endif

#ifndef MIN
#define MIN(a, b) (((a) < (b))? (a) : (b))
#endif

#define SAY_(fmt, ...) \
	do { fprintf(stderr, fmt "%.1s", __func__, __LINE__, __VA_ARGS__); } while (0)
#define SAY(...) SAY_(">>>> (%s:%d) " __VA_ARGS__, "\n")
#define HAI SAY("HAI")

/*
 * P R E - P R O C E S S I N G  R O U T I N E S
 *
 * Outputs an encoded normal form of the RFC1035 master file syntax.
 *
 * o Linear spaces and tabs trimmed.
 *
 * o Multi-line groups are folded, with grouping parentheses replaced by
 *   spaces.
 *
 * o Comments are discarded.
 *
 * o Quoted and escaped whitespace characters are coded as a short with the
 *   T_LIT bit set.
 *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#define T_LIT 0x0100

struct zonepp {
	int escaped, quoted, grouped, comment, lastc;
	struct { int v, n; } octet;
}; /* struct zonepp */

#define islwsp(ch) ((ch) == ' ' || (ch) == '\t')

static size_t fold(short *dst, size_t lim, unsigned char **src, size_t *len, struct zonepp *state) {
	short *p, *pe;
	int ch;

	if (!state->lastc)
		state->lastc = '\n';

	p  = dst;
	pe = p + lim;

	while (p < pe && *len) {
		ch = **src;

		if (state->escaped) {
			if (state->octet.n && (state->octet.n >= 3 || !isdigit(ch))) {
				state->escaped = 0;

				ch = state->octet.v;
				state->octet.v = 0;
				state->octet.n = 0;

				if (isspace(ch))
					ch |= T_LIT;

				*p++ = ch;

				continue; /* don't skip current char */
			} else if (isdigit(ch)) {
				state->octet.v *= 10;
				state->octet.v += ch - '0';
				state->octet.n++;

				goto skip;
			} else {
				state->escaped = 0;

				if (isspace(ch))
					ch |= T_LIT;

				goto copy;
			}
		} else if (state->comment) {
			if (ch != '\n')
				goto skip;

			state->comment = 0;
		}

test:
		switch (ch) {
		case '\\':
			state->escaped = 1;

			goto skip;
		case ';':
			if (state->quoted)
				break;

			state->comment = 1;

			goto skip;
		case '"':
			state->quoted = !state->quoted;

			goto skip;
		case '(':
			if (state->quoted || state->grouped)
				break;

			state->grouped = 1;

			ch = ' ';

			goto test;
		case ')':
			if (state->quoted || !state->grouped)
				break;

			state->grouped = 0;

			ch = ' ';

			goto test;
		case '\n':
			if (state->quoted) {
				break;
			} else if (state->lastc == '\n') {
				goto skip;
			} else if (state->grouped) {
				ch = ' ';

				goto test;
			}

			break;
		case '\t':
			ch = ' ';

			goto test;
		case ' ':
			if (state->quoted)
				break;
			else if (islwsp(state->lastc))
				goto skip;

			break;
		} /* switch() */
copy:
		if (state->quoted && isspace((0xff & ch)))
			ch |= T_LIT;

		state->lastc = *p = ch;

		++p;
skip:
		++*src;
		--*len;
	} /* while() */

	return p - dst;
} /* fold() */

struct tokens {
	short base[4096];
	unsigned count;
}; /* struct tokens */

static unsigned tok_eol(struct tokens *b) {
	unsigned i;

	for (i = 0; i < b->count; i++) {
		if (b->base[i] == '\n')
			return i + 1;
	}

	return 0;
} /* tok_eol() */

static void tok_discard(struct tokens *b, unsigned count) {
	if (count) {
		memmove(b->base, &b->base[count], 2 * (b->count - count));
		b->count = b->count - count;
	}
} /* tok_discard() */

struct zonefile {
	unsigned ttl;
	char origin[DNS_D_MAXNAME + 1];
	char lastrr[DNS_D_MAXNAME + 1];

	struct dns_soa soa;

	struct tokens toks;
	struct zonepp pp;
}; /* struct zonefile */

void zone_init(struct zonefile *P, const char *origin, unsigned ttl) {
	memset(P, 0, sizeof *P);

	origin = (origin)? origin : ".";
	dns_strlcpy(P->origin, origin, sizeof P->origin);
	dns_strlcpy(P->lastrr, origin, sizeof P->lastrr);

	P->ttl = (ttl)? ttl : 3600;
} /* zone_init() */

void zonerr_init(struct zonerr *rr, struct zonefile *P) {
	memset(rr, 0, sizeof *rr);
	rr->class = DNS_C_IN;
	rr->ttl   = P->ttl;
	dns_any_init(&rr->data, sizeof rr->data);
} /* zonerr_init() */

%%{
	machine file_grammar;
	alphtype short;

	action oops {
		fprintf(stderr, "OOPS: ");
		do {
			unsigned i, at = p - P->toks.base;
			for (i = 0; i < span; i++) {
				if (i == at) fputc('[', stderr);
				fputc((0xff & P->toks.base[i]), stderr);
				if (i == at) fputc(']', stderr);
			}
		} while(0);
		goto next;
	}

	action str_init { sp = str; }
	action str_copy { if (sp < endof(str)) *sp++ = fc; lc = fc; }
	action str_end { if (sp >= endof(str)) sp--; *sp = '\0'; }

	string = (any - space)+ >str_init $str_copy %str_end;

	number = digit+ >{ n = 0; } ${ n *= 10; n += fc - '0'; };

	ttl_unit =
		("w"i @{ ttl *= 604800; }) |
		("d"i @{ ttl *= 86400; }) |
		("h"i @{ ttl *= 3600; }) |
		("m"i @{ ttl *= 60; }) |
		("s"i);
	ttl = number %{ ttl = n; } ttl_unit? space*;

	action dom_end {
		 if (lc != '.') {
			if (P->origin[0] != '.')
				dns_strlcat(str, ".", sizeof str);
			dns_strlcat(str, P->origin, sizeof str);
		}
	}

	domain = string %dom_end;

	SOA_type    = "SOA"i %{ rr->type = DNS_T_SOA; };
	SOA_mname   = domain %{ dns_strlcpy(rr->data.soa.mname, str, sizeof rr->data.soa.mname); };
	SOA_rname   = string %{ dns_strlcpy(rr->data.soa.rname, str, sizeof rr->data.soa.rname); };
	SOA_serial  = number %{ rr->data.soa.serial = n; };
	SOA_refresh = ttl %{ rr->data.soa.refresh = ttl; };
	SOA_retry   = ttl %{ rr->data.soa.retry = ttl; };
	SOA_expire  = ttl %{ rr->data.soa.expire = ttl; };
	SOA_minimum = ttl %{ rr->data.soa.minimum = ttl; };
	SOA = SOA_type space+ SOA_mname space+ SOA_rname space+ SOA_serial space+ SOA_refresh space+ SOA_retry space+ SOA_expire space+ SOA_minimum space*;

	NS_type = "NS"i %{ rr->type = DNS_T_NS; };
	NS_host = domain %{ dns_strlcpy(rr->data.ns.host, str, sizeof rr->data.ns.host); };
	NS = NS_type space+ NS_host space*;

	MX_type = "MX"i %{ rr->type = DNS_T_MX; };
	MX_pref = number %{ rr->data.mx.preference = n; };
	MX_host = domain %{ dns_strlcpy(rr->data.mx.host, str, sizeof rr->data.mx.host); };
	MX = MX_type space+ MX_pref space+ MX_host space*;

	CNAME_type = "CNAME"i %{ rr->type = DNS_T_CNAME; };
	CNAME_host = domain %{ dns_strlcpy(rr->data.cname.host, str, sizeof rr->data.cname.host); };
	CNAME = CNAME_type space+ CNAME_host space*;

	SRV_type   = "SRV"i %{ rr->type = DNS_T_SRV; };
	SRV_pri    = number %{ rr->data.srv.priority = n; };
	SRV_weight = number %{ rr->data.srv.weight = n; };
	SRV_port   = number %{ rr->data.srv.port = n; };
	SRV_target = domain %{ dns_strlcpy(rr->data.srv.target, str, sizeof rr->data.srv.target); };
	SRV = SRV_type space+ SRV_pri space+ SRV_weight space+ SRV_port space+ SRV_target space*;

	# FIXME: Support multiple segments.
	TXT_type = "TXT"i %{ rr->type = DNS_T_TXT; };
	TXT_data = string %{ rr->data.txt.len = MIN(rr->data.txt.size - 1, sp - str); memcpy(rr->data.txt.data, str, rr->data.txt.len); };
	TXT = TXT_type space+ TXT_data space*;

	action AAAA_pton {
		if (1 != inet_pton(AF_INET6, str, &rr->data.aaaa.addr)) {
			SAY("invalid AAAA address: %s", str);
			goto next;
		}
	}

	AAAA_type   = "AAAA"i %{ rr->type = DNS_T_AAAA; };
	AAAA_digits = (xdigit | "." | ":")+ >str_init $str_copy %str_end;
	AAAA_addr   = AAAA_digits %AAAA_pton;
	AAAA = AAAA_type space+ AAAA_addr space*;

	A_type  = "A"i %{ rr->type = DNS_T_A; };
	A_octet = digit{1,3} >{ n = 0; } ${ n *= 10; n += fc - '0'; } %{ rr->data.a.addr.s_addr <<= 8; rr->data.a.addr.s_addr |= (0xff & n); };
	A_addr  = (A_octet "." A_octet "." A_octet "." A_octet) %{ rr->data.a.addr.s_addr = htonl(rr->data.a.addr.s_addr); };
	A = A_type space+ A_addr space*;

	PTR_type  = "PTR"i %{ rr->type = DNS_T_PTR; };
	PTR_cname = domain %{ dns_strlcpy(rr->data.ptr.host, str, sizeof rr->data.ptr.host); };
	PTR = PTR_type space+ PTR_cname space*;

	rrdata = (SOA | NS | MX | CNAME | SRV | TXT | AAAA | A | PTR) space*;

	rrname_blank  = " " %{ dns_strlcpy(rr->name, P->lastrr, sizeof rr->name); } " "*;
	rrname_origin = "@ " %{ dns_strlcpy(rr->name, P->origin, sizeof rr->name); } " "*;
	rrname_plain  = domain %{ if (!rr->name[0]) dns_strlcpy(rr->name, str, sizeof rr->name); } " "+;

	rrname = rrname_blank | rrname_origin | rrname_plain;

	rrclass = "IN " %{ rr->class = DNS_C_IN; };

	rrttl = ttl %{ rr->ttl = ttl; } space+;

	zonerr = rrname rrttl? rrclass? rrttl? rrdata;

	ORIGIN = "$ORIGIN " domain %{ dns_strlcpy(P->origin, str, sizeof P->origin); goto next; } space*;

	TTL    = "$TTL " ttl %{ P->ttl = ttl; goto next; } space*;

	nothing = space* %{ goto next; };

	main := (ORIGIN | TTL | zonerr | nothing) $!oops;
}%%

struct zonerr *zone_getrr(struct zonerr *rr, struct dns_soa **soa, struct zonefile *P) {
	%% write data;
	short *p, *pe, *eof;
	int cs;
	char str[1024], *sp, lc;
	unsigned span, ttl, n;

	span = 0;
next:
	tok_discard(&P->toks, span);

	if (!(span = tok_eol(&P->toks)))
		return NULL;

	sp  = str;
	lc  = 0;
	ttl = 0;
	n   = 0;

	zonerr_init(rr, P);

	%% write init;

	p   = P->toks.base;
	pe  = p + span;
	eof = pe;

	%% write exec;

	tok_discard(&P->toks, span);

	dns_strlcpy(P->lastrr, rr->name, sizeof P->lastrr);

	if (rr->type == DNS_T_SOA)
		P->soa = rr->data.soa;

	*soa = &P->soa;

	return rr;
} /* zone_getrr() */

size_t zone_parsesome(struct zonefile *P, const void *src, size_t len) {
	unsigned char *p = (unsigned char *)src;
	size_t lim = lengthof(P->toks.base) - P->toks.count;

	P->toks.count += fold(&P->toks.base[P->toks.count], lim, &p, &len, &P->pp);

	return p - (unsigned char *)src;
} /* zone_parsesome() */

size_t zone_parsefile(struct zonefile *P, FILE *fp) {
	unsigned char buf[1024];
	size_t tlim, lim, len;

	tlim = lengthof(P->toks.base) - P->toks.count;
	lim  = MIN(lengthof(buf), tlim);

	if (!(len = fread(buf, 1, lim, fp)))
		return 0;

	assert(len == zone_parsesome(P, buf, len));

	return len;
} /* zone_parsefile() */

struct zonefile *zone_open(const char *origin, unsigned ttl, int *error) {
	struct zonefile *P;

	if (!(P = malloc(sizeof *P)))
		{ *error = errno; return NULL; }

	zone_init(P, origin, ttl);

	return P;
} /* zone_open() */

void zone_close(struct zonefile *P) {
	free(P);
} /* zone_close() */

#if ZONE_MAIN

#include <stdlib.h>	/* EXIT_FAILURE */

#include <unistd.h>	/* getopt(3) */

struct {
	const char *progname;
} MAIN;

static void parsefile(FILE *fp, const char *origin, unsigned ttl) {
	struct zonefile P;
	struct zonerr rr;
	struct dns_soa *soa;

	zone_init(&P, origin, ttl);

	while (zone_parsefile(&P, fp)) {
		while (zone_getrr(&rr, &soa, &P)) {
			char data[512];
			dns_any_print(data, sizeof data, &rr.data, rr.type);
			printf("%s %u IN %s %s\n", rr.name, rr.ttl, dns_strtype(rr.type), data);
		}
	}
} /* parsefile() */

static void foldfile(FILE *dst, FILE *src) {
	struct zonepp state;
	unsigned char in[16], *p;
	short out[16];
	size_t len, num, i;

	memset(&state, 0, sizeof state);

	while ((len = fread(in, 1, sizeof in, src))) {
		p = in;

		while ((num = fold(out, sizeof out, &p, &len, &state))) {
			for (i = 0; i < num; i++) {
				if ((out[i] & T_LIT)
				||  (!isgraph(0xff & out[i]) && !isspace(0xff & out[i]))) {
					fprintf(dst, "\\%.3d", (0xff & out[i]));
				} else {
					fputc((0xff & out[i]), dst);
				}
			}
		}
	}
} /* foldfile() */

static void usage(FILE *fp) {
	static const char *usage =
		" [OPTIONS] [COMMAND]\n"
		"  -o ORIGIN  Zone origin\n"
		"  -t TTL     Zone TTL\n"
		"  -V         Print version info\n"
		"  -h         Print this usage message\n"
		"\n"
		"  fold       Fold into simple normal form\n"
		"  parse      Parse zone file and recompose\n"
		"\n"
		"Report bugs to William Ahern <william@25thandClement.com>\n";

	fputs(MAIN.progname, fp);
	fputs(usage, fp);
	fflush(fp);
} /* usage() */

static unsigned parsettl(const char *opt) {
	unsigned ttl;
	char *end;

	ttl = strtoul(opt, &end, 10);

	switch (tolower((unsigned char)*end)) {
	case 'w':
		ttl *= 7;
	case 'd':
		ttl *= 24;
	case 'h':
		ttl *= 60;
	case 'm':
		ttl *= 60;
	case 's':
		ttl *= 1;
	case '\0':
		break;
	default:
		fprintf(stderr, "%s: ", MAIN.progname);

		for (; *opt; opt++) {
			if (opt == end)
				fprintf(stderr, "[%c]", *opt);
			else
				fputc(*opt, stderr);
		}
		fputs(": invalid TTL\n", stderr);

		exit(EXIT_FAILURE);
	}

	return ttl;
} /* parsettl() */

#define CMD_FOLD  1
#define CMD_PARSE 2

static int command(const char *arg) {
	const char *p, *pe, *eof;
	int cs;

	arg = (arg && *arg)? arg : "parse";

	p   = arg;
	pe  = p + strlen(arg);
	eof = pe;

	%%{
		machine command;

		action oops {
			fprintf(stderr, "%s: %s: invalid command\n", MAIN.progname, arg);
			usage(stderr);
			exit(EXIT_FAILURE);
		}

		fold = "fold" %{ return CMD_FOLD; };
		parse = "parse" %{ return CMD_PARSE; };

		main := (fold | parse) $!oops;

		write data;
		write init;
		write exec;
	}%%

	return 0;
} /* command() */

int main(int argc, char **argv) {
	extern int optind;
	extern char *optarg;
	int opt;
	const char *origin = ".";
	unsigned ttl = 3600;

	MAIN.progname = argv[0];

	while (-1 != (opt = getopt(argc, argv, "o:t:Vh"))) {
		switch (opt) {
		case 'o':
			origin = optarg;

			break;
		case 't':
			ttl = parsettl(optarg);

			break;
		case 'h':
			usage(stdout);

			return 0;
		default:
			usage(stderr);

			return EXIT_FAILURE;
		} /* switch() */
	} /* getopt() */

	argc -= optind;
	argv += optind;

	switch (command(argv[0])) {
	case CMD_FOLD:
		foldfile(stdout, stdin);
		break;
	case CMD_PARSE:
		parsefile(stdin, origin, ttl);
		break;
	}

	return 0;
} /* main() */

#endif /* ZONE_MAIN */


Change separator to be more precise in what is matched

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  delimiter        = 0;
  cr               = "\r";
  lf               = "\n";
  separator        = ' ' | cr lf | lf;
  quote            = "'";
  double_quote     = '"';
  underscore       = '_';

  quoted_string    = quote (^quote | quote{2})** quote;

  unsigned_integer = digit+;

  identifier_start = alpha;
  identifier_part  = identifier_start | digit | double_quote{2};
  identifier_body  = identifier_start (underscore? identifier_part)*;
  identifier       = double_quote identifier_body double_quote;

  main := |*
    delimiter;
    separator;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    '%'              { emit('mod');         };
    '^'              { emit('pow');         };
    'E'              { emit('E');           };

    quoted_string    { emit_string();           };
    unsigned_integer { emit_unsigned_integer(); };

    identifier       { emit_identifier(); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE  = 1024
    CHUNK_SIZE  = 1  # XXX: used for testing
    END_OF_FILE = [0].freeze

    %% write data;

    def initialize(input)
      @input    = input
      @encoding = @input.external_encoding
      @block    = nil
      @buffer   = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      begin
        chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end until chunk == END_OF_FILE
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_identifier
      identifier = text
      identifier.sub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_string
      string = text
      string.sub!(/\A'(.*)'\z/, '\1') and string.gsub!("''", "'")
      emit(:string, string)
    end

    def text
      @buffer[@ts...@te].pack('C*').force_encoding(@encoding)
    end

    def read_chunk
      @input.read_nonblock(CHUNK_SIZE).bytes
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      END_OF_FILE
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  delimiter        = 0;
  separator        = ' ' | "\r" | "\n";
  quote            = "'";
  double_quote     = '"';
  underscore       = '_';

  quoted_string    = quote (^quote | quote{2})** quote;

  unsigned_integer = digit+;

  identifier_start = alpha;
  identifier_part  = identifier_start | digit | double_quote{2};
  identifier_body  = identifier_start (underscore? identifier_part)*;
  identifier       = double_quote identifier_body double_quote;

  main := |*
    delimiter;
    separator;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    '%'              { emit('mod');         };
    '^'              { emit('pow');         };
    'E'              { emit('E');           };

    quoted_string    { emit_string();           };
    unsigned_integer { emit_unsigned_integer(); };

    identifier       { emit_identifier(); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE  = 1024
    CHUNK_SIZE  = 1  # XXX: used for testing
    END_OF_FILE = [0].freeze

    %% write data;

    def initialize(input)
      @input    = input
      @encoding = @input.external_encoding
      @block    = nil
      @buffer   = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      begin
        chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end until chunk == END_OF_FILE
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_identifier
      identifier = text
      identifier.sub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_string
      string = text
      string.sub!(/\A'(.*)'\z/, '\1') and string.gsub!("''", "'")
      emit(:string, string)
    end

    def text
      @buffer[@ts...@te].pack('C*').force_encoding(@encoding)
    end

    def read_chunk
      @input.read_nonblock(CHUNK_SIZE).bytes
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      END_OF_FILE
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL


Type checking

Type checking

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:key_word,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A' ) >mark %{ emit.call(:value,data,ts,p) } );

  id        = ( 'ID=' id_value ) @kw;
  key_value = ( (id | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:key_word,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A' ) >mark %{ emit.call(:value,data,ts,p) } );

  id        = ( 'ID=' id_value ) @kw;
  key_value = ( (id | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  res = {}
  # p values
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Add documentation to state between parses

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    end
  }

  action consume_null {
    buffer.bytesize < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.bytesize < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer) >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size # how many bytes current message contains in total
        content_length = state.content_length # content length of current message

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    end
  }

  action consume_null {
    buffer.length < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.length < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer) >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size
        content_length = state.content_length

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end