Ragel

Ragel

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_]+)         >mark %{ emit("identifier",data,ts,p) };
  string      = (["][^"]*["])                >mark %{ emit("string",data,ts,p) };
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|string) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+           >mark %{ emit("identifier",data,ts,p) };
  string      = ["][^"]*["]                  >mark %{ emit("string",data,ts,p) };
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') %{ print "*k:" } );
  value       = ( (integer|float|boolean|identifier|string) %{ print "*v:" } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)


pmdastatsd: fix compile warnings in ragel generated code

Restore the void casts for statsd_en_main, statsd_error and
statsd_first_final to fix compiler warnings on rawhide. Only
the void cast for _statsd_eof_actions needs to be removed
because that variable is no longer generated with the newer
version of ragel on rawhide.

/*
 * Copyright (c) 2019 Miroslav Foltýn.  All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>

#include "parsers.h"
#include "parser-ragel.h"
#include "parsers-utils.h"
#include "utils.h"

%%{
	machine statsd;
	write data;
}%%

/**
 * Ragel parser entry point
 * Parsers given buffer and populates datagram with parsed data if they are valid
 * @arg str - Buffer to be parsed
 * @arg datagram - Placeholder for parsed data
 * @return 1 on success, 0 on fail
 */
int
ragel_parser_parse(char* str, struct statsd_datagram** datagram) {
	*datagram = (struct statsd_datagram*) malloc(sizeof(struct statsd_datagram));
	*(*datagram) = (struct statsd_datagram) {0};
	ALLOC_CHECK("Not enough memory to save datagram");
	size_t length = strlen(str);
	char *p = str, *pe = (str + length + 1);
	char *eof = pe;
	int cs;
	size_t current_index = 0;
	size_t current_segment_start_index = 0;
	struct tag_collection* tags = NULL;
	char* tag_key = NULL;
	char* tag_value = NULL;
	int tag_key_allocated = 0;
	int tag_value_allocated = 0;
	int any_tags = 0;

	%%{

		action error_handler {
			goto error_clean_up;
		}

		action not_final_state_finish {
			goto error_clean_up;
		}

		action increment_character_counter {
			current_index++;
		}

		action name_parsing {
			current_segment_start_index = current_index;
		}

		action name_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			(*datagram)->name = (char*) malloc(current_segment_length + 1);
			ALLOC_CHECK("Not enough memory to save metric name");
			memcpy(
				(*datagram)->name,
				&str[current_segment_start_index],
				current_segment_length
			);
			(*datagram)->name[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		action tag_parsed {
			size_t key_len = strlen(tag_key) + 1;
			size_t value_len = strlen(tag_value) + 1;
			struct tag* t = (struct tag*) malloc(sizeof(struct tag));
			ALLOC_CHECK("Unable to allocate memory for tag.");
			t->key = (char*) malloc(key_len);
			ALLOC_CHECK("Unable to allocate memory for tag key.");
			memcpy(t->key, tag_key, key_len);
			t->value = (char*) malloc(value_len);
			ALLOC_CHECK("Unable to allocate memory for tag value.");
			memcpy(t->value, tag_value, value_len);
			if (!any_tags) {
				tags = (struct tag_collection*) malloc(sizeof(struct tag_collection));
				ALLOC_CHECK("Unable to allocate memory for tag collection.");
				tags->values = (struct tag**) malloc(sizeof(struct tag*));
				tags->values[0] = t;
				tags->length = 1;
				any_tags = 1;
			} else {
				struct tag** new_tags =
					(struct tag**) realloc(tags->values, sizeof(struct tag*) * (tags->length + 1));
				ALLOC_CHECK("Unable to allocate memory for tags");
				if (tags != NULL) {
					tags->values = new_tags;
					tags->values[tags->length] = t;
					tags->length++;
				}
			}
			free(tag_key);
			free(tag_value);
			tag_key = NULL;
			tag_value = NULL;
			tag_key_allocated = 0;
			tag_value_allocated = 0;
		}

		action value_parsed {
			char* startptr;
			char* endptr;
			if (str[current_segment_start_index] == '+') {
				(*datagram)->explicit_sign = SIGN_PLUS;
				startptr = &str[current_segment_start_index + 1];
			} else if (str[current_segment_start_index] == '-') {
				(*datagram)->explicit_sign = SIGN_MINUS;
				startptr = &str[current_segment_start_index + 1];
			} else {
				(*datagram)->explicit_sign = SIGN_NONE;
				startptr = &str[current_segment_start_index];
			}
			double value = strtod(startptr, &endptr);
			if (startptr == endptr || errno == ERANGE) {
				goto error_clean_up;
			}
			(*datagram)->value = value;
			current_segment_start_index = current_index + 1;
		}

		action type_parsed {
			if (str[current_segment_start_index] == 'c') {
				(*datagram)->type = METRIC_TYPE_COUNTER;
			} else if (str[current_segment_start_index] == 'g') {
				(*datagram)->type = METRIC_TYPE_GAUGE;
			} else {
				(*datagram)->type = METRIC_TYPE_DURATION;
			}
			current_segment_start_index = current_index + 1;
		}

		action tag_key_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			tag_key = (char *) realloc(tag_key, current_segment_length + 1);
			ALLOC_CHECK("Not enough memory for tag key buffer.");
			tag_key_allocated = 1;
			memcpy(
				tag_key,
				&str[current_segment_start_index],
				current_segment_length
			);
			tag_key[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		action tag_value_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			tag_value = (char *) realloc(tag_value, current_segment_length + 1);
			ALLOC_CHECK("Not enough memory for tag key buffer.");
			tag_value_allocated = 1;
			memcpy(
				tag_value,
				&str[current_segment_start_index],
				current_segment_length
			);
			tag_value[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		str_value = [a-z][a-zA-Z0-9_.]*;
		tag_string = [a-zA-Z0-9_.]{1,};
		name = str_value[:,]{1};
		value = ('+'|'-')?[0-9]+(('.')[0-9]+)?(('e'|'E')('+'|'-')?[0-9]+)?('|');
		type = ('c'|'g'|'ms')('@'|'\0'|'\n\0'|("|#"));
		tags = ((tag_string'=')@tag_key_parsed(tag_string(':'|',')@tag_value_parsed))+;
		tags_end = ((tag_string':')@tag_key_parsed(tag_string(','|'\0'|'\n\0')@tag_value_parsed))+;
		main := ((name >name_parsing @name_parsed . (tags @ tag_parsed)? . value @value_parsed . type @type_parsed . (tags_end @ tag_parsed)? ))
			  $~increment_character_counter @!error_handler @/not_final_state_finish;
		# */
		# Initialize and execute.
		write init;
		write exec;

	}%%
	(void)statsd_en_main;
	(void)statsd_error;
	(void)statsd_first_final;

	if (any_tags) {
		char* json = tag_collection_to_json(tags);
		if (json != NULL) {
			(*datagram)->tags = json;
			(*datagram)->tags_pair_count = tags->length;
		}
		free_tag_collection(tags);
		if (tag_key_allocated) free(tag_key);
		if (tag_value_allocated) free(tag_value);
	}
	if (str[length - 1] == '\n')
        str[length - 1] = 0;
	VERBOSE_LOG(2, "Parsed: %s", str);
	return 1;

	error_clean_up:
	if (any_tags) {
		free_tag_collection(tags);
	}
	if (tag_key_allocated) free(tag_key);
	if (tag_value_allocated) free(tag_value);
	if (str[length - 1] == '\n')
        str[length - 1] = 0;
	free_datagram(*datagram);
	METRIC_PROCESSING_ERR_LOG("Throwing away datagram. REASON: unable to parse: %s", str);
	return 0;
};

/**
 * --------------------------------------
 * |                                    |
 * |     UNIT TEST FOR THIS FILE        |
 * |                                    |
 * --------------------------------------
 */
#if _TEST_TARGET == 2

int
main() {
    INIT_TEST("Running tests for ragel parser:", ragel_parser_parse);
    SUITE_HEADER("Unparsable values")
    CHECK_ERROR("", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow:2", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow|g", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("2|g", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("1:1|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("e x-2 ple:20|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("example,tags=dwq=qwddqwd=qwd:10|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    SUITE_HEADER("Basic values");
    CHECK_ERROR("example:-1|c", "example", NULL, 1, METRIC_TYPE_COUNTER, SIGN_MINUS);
    CHECK_ERROR("example:+1|g", "example", NULL, 1, METRIC_TYPE_GAUGE, SIGN_PLUS);
    CHECK_ERROR("example:1|ms", "example", NULL, 1, METRIC_TYPE_DURATION, SIGN_NONE);
    SUITE_HEADER("Sanitizable metric name");
    SUITE_HEADER("Non integer values")
    CHECK_ERROR("example:1.2|c", "example", NULL, 1.2, METRIC_TYPE_COUNTER, SIGN_NONE);
    CHECK_ERROR("example:1.000000004|c", "example", NULL, 1.000000004, METRIC_TYPE_COUNTER, SIGN_NONE);
    CHECK_ERROR("example:1.00000000000000000000000000000004|c", "example", NULL, 1.00000000000000000000000000000004, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Instance descriptors")
    CHECK_ERROR("example,instance=20:20|c", "example", "{\"instance\":\"20\"}", 20, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Tag descriptors")
    CHECK_ERROR("example,tagY=20,tagX=10:10|c", "example", "{\"tagX\":\"10\",\"tagY\":\"20\"}", 10, METRIC_TYPE_COUNTER, SIGN_NONE);
	CHECK_ERROR("example,tagY=20:10|c|#tagY:20,tagW:W", "example", "{\"tagW\":\"W\",\"tagY\":\"20\"}", 10, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Malformed tags")
    SUITE_HEADER("Too long tag descriptor (gets thrown away)")
    CHECK_ERROR("example,tags=IhTicIzMhKsYSTiamskyBePkjZhgAFW6Gt97AAq3hbKrfs2Qrcf57NPMrjn3dzCaOcslkO8SU4hEQRDdlXFWs8foWCHJOMqbgoiSZlrKFHeO1sxNOimc5PqLswhWCxuF7M8v4ivySmHdIPxUxTk5Pq0PHtzIPnuJyYlrsT1jG2ZyF7Y5k2XIq5ZbSSQDxICPr6WvqsDVLEZofPXZydVpJ17nN7Xwb1whud6sniTGTelC8Y2hiXLB6GpA3oNPkSWrtwGP7mEm3FcLjPiKoQtTLWJd47X3krHs79cV0MduDmvCsPT9t6ojTDlg8u3emrv69DDLGLMNZpXTeorA5Yuiwqia5EHVPFGvZvXMGpRzkmBT1Jqu9J9PQj4ffkGTncZS3WDBmUoV7a2miMMLeEQbeTGG3F8b2OkALCSnARkBLePRVgsd3gOpPtFC4JcaiYrHKtpf4yb0QkWL0uSHBPV0GsJztlE4OmmVuCJwY6Lr8fVcG0V8iXkkRJPBcnKJu3Aim22y97jETteaveB8fnqZVt2WrF0ElyaMe5IkDpExwKCn09OMxSf8cDWwu00P8n06rbUmrcUh41r0ibAptOim1kxuV6SyYPnyBjqxT3QTM04kHZ0t0cww9uuxLdaGpyTi9Wzq5kmDnKBrX35jxvIkBIx53uFCzHfqDQc7EzoYEWuinaWghLtudErOPxd3YGKAeXa2R1hTRfEOBsgq05ldrDM4KkaAqOD4YimkHuIso1r5qS1KzFYiXCvwLZfxMdQK5x1JIiD9KIg4RTjQJgnTbC8OXgBfzI6GSkv03fNhtMYj2SYn6txhJL8yzzAQhbIdVLsiWJgCa0hAu9mocd98gGhYTqpWHJRp6E9F7Nt3ANWYSvOtaCYLdAgSGXWtSgcy74okd5si8dYcnSTY3BF3BJFxP8zOdO9Sp7EDkfVxrod5J7AIovdfTTlgY31qP0irJ1MCgxr3ZZToEknUFbnpVBFA0niWo721uyinVlVZH2ExzqmFUS1HsPqKUHt6YFNiJGOp3Y1QPC9jNmrUu35ssDt3W86RwTCq3VYqscvsd6NRDXoCEI1XQXH8RngdanNNSXnFwOFcSTKkM84WHy68e6GBsy5w4jVq7s1UoHxQfADOsO6RDkK6J3nR2dcQL0HCZ2tGoexgRVpxtAxiOI33njTcDUy5zYhJPutsbxFQgf7Nh0cqTwKTd8q8T3wScLfTp84jmQvEpyxNf0Ums6cgN4ttY8G5O1ilegsSYmxWA7Mq9mwmRtMymm1s7OkPL4TblkSmzrPDPdDgp2sRf7ETDAC2CzZv4cJMIyENHfi0N2zgSf5cmIW0a1W5mJlKUjuAoG7dxC8QxpkewEQt9B0ygxnA7MT67rpWblZcVqYz4jtpcMa15BeWExtY15UvkuZEneAC4TChMy3DJqzJujisFLKFnJKmRH5qsDRZDmYjN8UAkz3WQI4qR7PwPLOHr64qvumDmzXODlo7nPKH5mht5NsqSc1FMQ2J7oerZnz63sNC7Otnu2kzp9uVmbZqnEYZDyqtVCNUMG8utFQtFIPNgO1TjlexQuwgz1pSGP1ipsS1KYpFTEjlbYH1NJF3hYBDrCHZgeXAULmDLjWsUdKrtsnYTmisixoL4ovOp9NmWbMlYjW1sVC4GOKy9Ah2A8UHf5LzFsiBwo0hnS1B43WZ0lCn1e5vpE5EoOg94uysQpv1z4li15fXR9qNWKsc3cLR2JTLz4gIzrgwkdUtzdZ2oRCqAyT5TJg3JhaBNiBaodK4q3fjiiusQxFFDb5U5ptiNiDNTUiYKED1i9N5Ek3F962jv53Kbkcy7Ebi5Fu795RZz2vPigrqtHhUCe9V6qMRkGD5nmLFMxrgfHk4z4BMV1PDQdnz2ILlgHRFCsdHMjwhIxJxqS657heMpSmnq482Yxxb4NOmx0QopYTkwqZTv2FMN7M6Shr3yLYrkYxgCLuX7st59PfCORtU6L6IYe1sJpw7s1vfJcGh7mePmhVrEvc4AnXGMPYImOgXYTHhtxPxUFgfvXPpObhz1z8O74eo0Psdh3xVAOsatA4Sf2ufLNmwTWutaWyqeKiqrSDgxRtQ2gGEX0ZCZb0ZXvfycw06TQwECdu5XaTSKXSbV7VXO0Hg4b964uiziBEfmjB7EmDkpSTPIhFZJCktQyEfCRr4TYvPLY3RlAehHjR55XZvci2mtYBczVCfTKLfJvxj9XlfjYbth31RtpekbupV3AWL02PXkga7Md3MuLO7yLEKp8YNMpq5O0XNl2pdc4vhehy7sK4XEeViOslfWDRlDqeFMGPLnwHxgw4QFyu51DUrV8KCeo5sQ8CNiOKYstAd8Zb0ayfS6osmL5R2SLdWNECpgszhsa8g3NWYWvLVH2MzFzVHFx3JLTIzqCLJ0AwX070aNBCIezjdVtP44xn9JvfPJwU25lYfL3SvA154ElLvPUKmA1k4BeY7GfQY6rTD8P0jU8B4tPVK4k1wUc1nssslvFJ4fRbreGRVORLSeUiCDrD69IrjaJmAh0wIiFOCOuWMqU3NYdYT7Yr4fcpSmgpECnZQwv5HWzEk88DxakLaKGWPGp6zzAgt6D8MpxsN0O4kl6G7FATm52aW5tRnimbExpjLfRLrKOK656DvU6x2AOLGFr88x4Zg0xloSRZvGzuPkTWkjPrwKxUVfM8vGX6UrjkU8nxiP2TbspbPkiIoN0MgZxQFvrsYHqgFP4jxSO71IO6YaH1a6zj61d2hNJSWtx0nkwxoNTPdk061Sz2NSvKkYJLMmw5reeen5BMboRQHXBXMKeIgrwhjeQSQfQT6IJhNAeiDYZ9qM1DRGLiIakocwxQFNAd0qFeDcukkHkdviz4rbqaslcXQgrW6HCpeLd0LCD3MpftY7bJcBjlyOag9mD6I7apCWLUtm8jo5Wdv6bATUXwKFTxmeYgfjonoka4ynLjKw0K8olRmOZTBFuQeiTIUUAVW9yv9piqkKj6y2tmNxYRrdDLFwfHF361wln3v7EF5qDOtza1BLiGqwXfbugVqs9GcMQITtqP7T6Ysq1It5qz9QfUJ5nPC634FhYgfkpp8nGI94XsBmVSfDN6enWJWHz8E8nhSbqqRB75pisTQACU1dP49V0e2svcljkIa7T74i5EarkJJirLhFt7pK4ddNYN33J5MgHiIvFuEdJCZBX6sS6cVwMyO0QIBRVKq828BgoZ164JcFGXFKvBqdhy7WOFrg1ioYEJABTwOy4YxRhyJRAY9u1zDAy9F0j2FAk1Lx81V3CZLlsXGn1G16xQD9eYQn1tlvth9aKWO6SkecYkSk9Oj6ul9EUu19zSewmzWyv4ujWzPfHOEM0JFsj5mzAT4wN74swMJuu0ktdegItCooEopSPGPMfhnhw8vl3xxgQQLEe1B54WSiUtEWmOoQ3K7VncyNcLQPr1QhOHuOi4wziJ6dz5LsN06SngfNMcWcN1qouW5gjOxAzaXbO0oloHl5Y500Dc01YDRQbJjY8t33Co4fPE5vwS9kKpz2wpxnww2K89iVYQACL83FQu9jvS4PIWdgpLrLCRfBGjYmU1Qpgg5k0plUpDUUc36cI0U2XLeZrScvPL3jjc6tF1IRRuUALSniAwsyxjqr2UfWrsR0VzlFROVzydOh8VcGUGY03MUQEk1yMMMqg51lGsU0kFfVzrYfbjn3UjESDRnEBB4GUDqwHTdU8TQaOrFSBO0H0CyKiutq4WMPpVaZ1LQEmNx0WmCHX9QxyWb2woKKG5jFIpt3Bp6UmHwUBCYYalK5zRg0pKEHk3VOecAUj6sq0qaoCjdesbLnY9pm7ozi0GmAFftkucDpRX9NZZvjNvV6qbmLyM9oC3b2bezfZlMV41smqr0W1vOvgG4BMfC5ZMvUDXOM1wRknyeOyOFSxTcMpSuljO2vUUjOxg7rYiy9BK6MjJnbwaKyO5JZ8MidPWpvMJb16iAv8FwpTJr8xYSdg3EdSfQBPCrC9LmyBXJDIqxa0V9Qcm9Ee2r1lmfIsYH2uagRkcIJ4P8Ub0nJfEbG2WGPwN8q9YnPWpV1sZ2F0Gh6VI7yzp2rQYZL6rXh8j4jiSLGl1vaxj7:10|c", "example", NULL, 10, METRIC_TYPE_COUNTER, SIGN_NONE);
    END_TEST();
}

#endif

/*
 * Copyright (c) 2019 Miroslav Foltýn.  All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>

#include "parsers.h"
#include "parser-ragel.h"
#include "parsers-utils.h"
#include "utils.h"

%%{
	machine statsd;
	write data;
}%%

/**
 * Ragel parser entry point
 * Parsers given buffer and populates datagram with parsed data if they are valid
 * @arg str - Buffer to be parsed
 * @arg datagram - Placeholder for parsed data
 * @return 1 on success, 0 on fail
 */
int
ragel_parser_parse(char* str, struct statsd_datagram** datagram) {
	*datagram = (struct statsd_datagram*) malloc(sizeof(struct statsd_datagram));
	*(*datagram) = (struct statsd_datagram) {0};
	ALLOC_CHECK("Not enough memory to save datagram");
	size_t length = strlen(str);
	char *p = str, *pe = (str + length + 1);
	char *eof = pe;
	int cs;
	size_t current_index = 0;
	size_t current_segment_start_index = 0;
	struct tag_collection* tags = NULL;
	char* tag_key = NULL;
	char* tag_value = NULL;
	int tag_key_allocated = 0;
	int tag_value_allocated = 0;
	int any_tags = 0;

	%%{

		action error_handler {
			goto error_clean_up;
		}

		action not_final_state_finish {
			goto error_clean_up;
		}

		action increment_character_counter {
			current_index++;
		}

		action name_parsing {
			current_segment_start_index = current_index;
		}

		action name_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			(*datagram)->name = (char*) malloc(current_segment_length + 1);
			ALLOC_CHECK("Not enough memory to save metric name");
			memcpy(
				(*datagram)->name,
				&str[current_segment_start_index],
				current_segment_length
			);
			(*datagram)->name[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		action tag_parsed {
			size_t key_len = strlen(tag_key) + 1;
			size_t value_len = strlen(tag_value) + 1;
			struct tag* t = (struct tag*) malloc(sizeof(struct tag));
			ALLOC_CHECK("Unable to allocate memory for tag.");
			t->key = (char*) malloc(key_len);
			ALLOC_CHECK("Unable to allocate memory for tag key.");
			memcpy(t->key, tag_key, key_len);
			t->value = (char*) malloc(value_len);
			ALLOC_CHECK("Unable to allocate memory for tag value.");
			memcpy(t->value, tag_value, value_len);
			if (!any_tags) {
				tags = (struct tag_collection*) malloc(sizeof(struct tag_collection));
				ALLOC_CHECK("Unable to allocate memory for tag collection.");
				tags->values = (struct tag**) malloc(sizeof(struct tag*));
				tags->values[0] = t;
				tags->length = 1;
				any_tags = 1;
			} else {
				struct tag** new_tags =
					(struct tag**) realloc(tags->values, sizeof(struct tag*) * (tags->length + 1));
				ALLOC_CHECK("Unable to allocate memory for tags");
				if (tags != NULL) {
					tags->values = new_tags;
					tags->values[tags->length] = t;
					tags->length++;
				}
			}
			free(tag_key);
			free(tag_value);
			tag_key = NULL;
			tag_value = NULL;
			tag_key_allocated = 0;
			tag_value_allocated = 0;
		}

		action value_parsed {
			char* startptr;
			char* endptr;
			if (str[current_segment_start_index] == '+') {
				(*datagram)->explicit_sign = SIGN_PLUS;
				startptr = &str[current_segment_start_index + 1];
			} else if (str[current_segment_start_index] == '-') {
				(*datagram)->explicit_sign = SIGN_MINUS;
				startptr = &str[current_segment_start_index + 1];
			} else {
				(*datagram)->explicit_sign = SIGN_NONE;
				startptr = &str[current_segment_start_index];
			}
			double value = strtod(startptr, &endptr);
			if (startptr == endptr || errno == ERANGE) {
				goto error_clean_up;
			}
			(*datagram)->value = value;
			current_segment_start_index = current_index + 1;
		}

		action type_parsed {
			if (str[current_segment_start_index] == 'c') {
				(*datagram)->type = METRIC_TYPE_COUNTER;
			} else if (str[current_segment_start_index] == 'g') {
				(*datagram)->type = METRIC_TYPE_GAUGE;
			} else {
				(*datagram)->type = METRIC_TYPE_DURATION;
			}
			current_segment_start_index = current_index + 1;
		}

		action tag_key_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			tag_key = (char *) realloc(tag_key, current_segment_length + 1);
			ALLOC_CHECK("Not enough memory for tag key buffer.");
			tag_key_allocated = 1;
			memcpy(
				tag_key,
				&str[current_segment_start_index],
				current_segment_length
			);
			tag_key[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		action tag_value_parsed {
			size_t current_segment_length = current_index - current_segment_start_index;
			tag_value = (char *) realloc(tag_value, current_segment_length + 1);
			ALLOC_CHECK("Not enough memory for tag key buffer.");
			tag_value_allocated = 1;
			memcpy(
				tag_value,
				&str[current_segment_start_index],
				current_segment_length
			);
			tag_value[current_segment_length] = '\0';
			current_segment_start_index = current_index + 1;
		}

		str_value = [a-z][a-zA-Z0-9_.]*;
		tag_string = [a-zA-Z0-9_.]{1,};
		name = str_value[:,]{1};
		value = ('+'|'-')?[0-9]+(('.')[0-9]+)?(('e'|'E')('+'|'-')?[0-9]+)?('|');
		type = ('c'|'g'|'ms')('@'|'\0'|'\n\0'|("|#"));
		tags = ((tag_string'=')@tag_key_parsed(tag_string(':'|',')@tag_value_parsed))+;
		tags_end = ((tag_string':')@tag_key_parsed(tag_string(','|'\0'|'\n\0')@tag_value_parsed))+;
		main := ((name >name_parsing @name_parsed . (tags @ tag_parsed)? . value @value_parsed . type @type_parsed . (tags_end @ tag_parsed)? ))
			  $~increment_character_counter @!error_handler @/not_final_state_finish;
		# */
		# Initialize and execute.
		write init;
		write exec;

	}%%

	if (any_tags) {
		char* json = tag_collection_to_json(tags);
		if (json != NULL) {
			(*datagram)->tags = json;
			(*datagram)->tags_pair_count = tags->length;
		}
		free_tag_collection(tags);
		if (tag_key_allocated) free(tag_key);
		if (tag_value_allocated) free(tag_value);
	}
	if (str[length - 1] == '\n')
        str[length - 1] = 0;
	VERBOSE_LOG(2, "Parsed: %s", str);
	return 1;

	error_clean_up:
	if (any_tags) {
		free_tag_collection(tags);
	}
	if (tag_key_allocated) free(tag_key);
	if (tag_value_allocated) free(tag_value);
	if (str[length - 1] == '\n')
        str[length - 1] = 0;
	free_datagram(*datagram);
	METRIC_PROCESSING_ERR_LOG("Throwing away datagram. REASON: unable to parse: %s", str);
	return 0;
};

/**
 * --------------------------------------
 * |                                    |
 * |     UNIT TEST FOR THIS FILE        |
 * |                                    |
 * --------------------------------------
 */
#if _TEST_TARGET == 2

int
main() {
    INIT_TEST("Running tests for ragel parser:", ragel_parser_parse);
    SUITE_HEADER("Unparsable values")
    CHECK_ERROR("", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow:2", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("wow|g", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("2|g", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("1:1|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("e x-2 ple:20|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    CHECK_ERROR("example,tags=dwq=qwddqwd=qwd:10|c", NULL, NULL, 0, METRIC_TYPE_NONE, SIGN_NONE);
    SUITE_HEADER("Basic values");
    CHECK_ERROR("example:-1|c", "example", NULL, 1, METRIC_TYPE_COUNTER, SIGN_MINUS);
    CHECK_ERROR("example:+1|g", "example", NULL, 1, METRIC_TYPE_GAUGE, SIGN_PLUS);
    CHECK_ERROR("example:1|ms", "example", NULL, 1, METRIC_TYPE_DURATION, SIGN_NONE);
    SUITE_HEADER("Sanitizable metric name");
    SUITE_HEADER("Non integer values")
    CHECK_ERROR("example:1.2|c", "example", NULL, 1.2, METRIC_TYPE_COUNTER, SIGN_NONE);
    CHECK_ERROR("example:1.000000004|c", "example", NULL, 1.000000004, METRIC_TYPE_COUNTER, SIGN_NONE);
    CHECK_ERROR("example:1.00000000000000000000000000000004|c", "example", NULL, 1.00000000000000000000000000000004, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Instance descriptors")
    CHECK_ERROR("example,instance=20:20|c", "example", "{\"instance\":\"20\"}", 20, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Tag descriptors")
    CHECK_ERROR("example,tagY=20,tagX=10:10|c", "example", "{\"tagX\":\"10\",\"tagY\":\"20\"}", 10, METRIC_TYPE_COUNTER, SIGN_NONE);
	CHECK_ERROR("example,tagY=20:10|c|#tagY:20,tagW:W", "example", "{\"tagW\":\"W\",\"tagY\":\"20\"}", 10, METRIC_TYPE_COUNTER, SIGN_NONE);
    SUITE_HEADER("Malformed tags")
    SUITE_HEADER("Too long tag descriptor (gets thrown away)")
    CHECK_ERROR("example,tags=IhTicIzMhKsYSTiamskyBePkjZhgAFW6Gt97AAq3hbKrfs2Qrcf57NPMrjn3dzCaOcslkO8SU4hEQRDdlXFWs8foWCHJOMqbgoiSZlrKFHeO1sxNOimc5PqLswhWCxuF7M8v4ivySmHdIPxUxTk5Pq0PHtzIPnuJyYlrsT1jG2ZyF7Y5k2XIq5ZbSSQDxICPr6WvqsDVLEZofPXZydVpJ17nN7Xwb1whud6sniTGTelC8Y2hiXLB6GpA3oNPkSWrtwGP7mEm3FcLjPiKoQtTLWJd47X3krHs79cV0MduDmvCsPT9t6ojTDlg8u3emrv69DDLGLMNZpXTeorA5Yuiwqia5EHVPFGvZvXMGpRzkmBT1Jqu9J9PQj4ffkGTncZS3WDBmUoV7a2miMMLeEQbeTGG3F8b2OkALCSnARkBLePRVgsd3gOpPtFC4JcaiYrHKtpf4yb0QkWL0uSHBPV0GsJztlE4OmmVuCJwY6Lr8fVcG0V8iXkkRJPBcnKJu3Aim22y97jETteaveB8fnqZVt2WrF0ElyaMe5IkDpExwKCn09OMxSf8cDWwu00P8n06rbUmrcUh41r0ibAptOim1kxuV6SyYPnyBjqxT3QTM04kHZ0t0cww9uuxLdaGpyTi9Wzq5kmDnKBrX35jxvIkBIx53uFCzHfqDQc7EzoYEWuinaWghLtudErOPxd3YGKAeXa2R1hTRfEOBsgq05ldrDM4KkaAqOD4YimkHuIso1r5qS1KzFYiXCvwLZfxMdQK5x1JIiD9KIg4RTjQJgnTbC8OXgBfzI6GSkv03fNhtMYj2SYn6txhJL8yzzAQhbIdVLsiWJgCa0hAu9mocd98gGhYTqpWHJRp6E9F7Nt3ANWYSvOtaCYLdAgSGXWtSgcy74okd5si8dYcnSTY3BF3BJFxP8zOdO9Sp7EDkfVxrod5J7AIovdfTTlgY31qP0irJ1MCgxr3ZZToEknUFbnpVBFA0niWo721uyinVlVZH2ExzqmFUS1HsPqKUHt6YFNiJGOp3Y1QPC9jNmrUu35ssDt3W86RwTCq3VYqscvsd6NRDXoCEI1XQXH8RngdanNNSXnFwOFcSTKkM84WHy68e6GBsy5w4jVq7s1UoHxQfADOsO6RDkK6J3nR2dcQL0HCZ2tGoexgRVpxtAxiOI33njTcDUy5zYhJPutsbxFQgf7Nh0cqTwKTd8q8T3wScLfTp84jmQvEpyxNf0Ums6cgN4ttY8G5O1ilegsSYmxWA7Mq9mwmRtMymm1s7OkPL4TblkSmzrPDPdDgp2sRf7ETDAC2CzZv4cJMIyENHfi0N2zgSf5cmIW0a1W5mJlKUjuAoG7dxC8QxpkewEQt9B0ygxnA7MT67rpWblZcVqYz4jtpcMa15BeWExtY15UvkuZEneAC4TChMy3DJqzJujisFLKFnJKmRH5qsDRZDmYjN8UAkz3WQI4qR7PwPLOHr64qvumDmzXODlo7nPKH5mht5NsqSc1FMQ2J7oerZnz63sNC7Otnu2kzp9uVmbZqnEYZDyqtVCNUMG8utFQtFIPNgO1TjlexQuwgz1pSGP1ipsS1KYpFTEjlbYH1NJF3hYBDrCHZgeXAULmDLjWsUdKrtsnYTmisixoL4ovOp9NmWbMlYjW1sVC4GOKy9Ah2A8UHf5LzFsiBwo0hnS1B43WZ0lCn1e5vpE5EoOg94uysQpv1z4li15fXR9qNWKsc3cLR2JTLz4gIzrgwkdUtzdZ2oRCqAyT5TJg3JhaBNiBaodK4q3fjiiusQxFFDb5U5ptiNiDNTUiYKED1i9N5Ek3F962jv53Kbkcy7Ebi5Fu795RZz2vPigrqtHhUCe9V6qMRkGD5nmLFMxrgfHk4z4BMV1PDQdnz2ILlgHRFCsdHMjwhIxJxqS657heMpSmnq482Yxxb4NOmx0QopYTkwqZTv2FMN7M6Shr3yLYrkYxgCLuX7st59PfCORtU6L6IYe1sJpw7s1vfJcGh7mePmhVrEvc4AnXGMPYImOgXYTHhtxPxUFgfvXPpObhz1z8O74eo0Psdh3xVAOsatA4Sf2ufLNmwTWutaWyqeKiqrSDgxRtQ2gGEX0ZCZb0ZXvfycw06TQwECdu5XaTSKXSbV7VXO0Hg4b964uiziBEfmjB7EmDkpSTPIhFZJCktQyEfCRr4TYvPLY3RlAehHjR55XZvci2mtYBczVCfTKLfJvxj9XlfjYbth31RtpekbupV3AWL02PXkga7Md3MuLO7yLEKp8YNMpq5O0XNl2pdc4vhehy7sK4XEeViOslfWDRlDqeFMGPLnwHxgw4QFyu51DUrV8KCeo5sQ8CNiOKYstAd8Zb0ayfS6osmL5R2SLdWNECpgszhsa8g3NWYWvLVH2MzFzVHFx3JLTIzqCLJ0AwX070aNBCIezjdVtP44xn9JvfPJwU25lYfL3SvA154ElLvPUKmA1k4BeY7GfQY6rTD8P0jU8B4tPVK4k1wUc1nssslvFJ4fRbreGRVORLSeUiCDrD69IrjaJmAh0wIiFOCOuWMqU3NYdYT7Yr4fcpSmgpECnZQwv5HWzEk88DxakLaKGWPGp6zzAgt6D8MpxsN0O4kl6G7FATm52aW5tRnimbExpjLfRLrKOK656DvU6x2AOLGFr88x4Zg0xloSRZvGzuPkTWkjPrwKxUVfM8vGX6UrjkU8nxiP2TbspbPkiIoN0MgZxQFvrsYHqgFP4jxSO71IO6YaH1a6zj61d2hNJSWtx0nkwxoNTPdk061Sz2NSvKkYJLMmw5reeen5BMboRQHXBXMKeIgrwhjeQSQfQT6IJhNAeiDYZ9qM1DRGLiIakocwxQFNAd0qFeDcukkHkdviz4rbqaslcXQgrW6HCpeLd0LCD3MpftY7bJcBjlyOag9mD6I7apCWLUtm8jo5Wdv6bATUXwKFTxmeYgfjonoka4ynLjKw0K8olRmOZTBFuQeiTIUUAVW9yv9piqkKj6y2tmNxYRrdDLFwfHF361wln3v7EF5qDOtza1BLiGqwXfbugVqs9GcMQITtqP7T6Ysq1It5qz9QfUJ5nPC634FhYgfkpp8nGI94XsBmVSfDN6enWJWHz8E8nhSbqqRB75pisTQACU1dP49V0e2svcljkIa7T74i5EarkJJirLhFt7pK4ddNYN33J5MgHiIvFuEdJCZBX6sS6cVwMyO0QIBRVKq828BgoZ164JcFGXFKvBqdhy7WOFrg1ioYEJABTwOy4YxRhyJRAY9u1zDAy9F0j2FAk1Lx81V3CZLlsXGn1G16xQD9eYQn1tlvth9aKWO6SkecYkSk9Oj6ul9EUu19zSewmzWyv4ujWzPfHOEM0JFsj5mzAT4wN74swMJuu0ktdegItCooEopSPGPMfhnhw8vl3xxgQQLEe1B54WSiUtEWmOoQ3K7VncyNcLQPr1QhOHuOi4wziJ6dz5LsN06SngfNMcWcN1qouW5gjOxAzaXbO0oloHl5Y500Dc01YDRQbJjY8t33Co4fPE5vwS9kKpz2wpxnww2K89iVYQACL83FQu9jvS4PIWdgpLrLCRfBGjYmU1Qpgg5k0plUpDUUc36cI0U2XLeZrScvPL3jjc6tF1IRRuUALSniAwsyxjqr2UfWrsR0VzlFROVzydOh8VcGUGY03MUQEk1yMMMqg51lGsU0kFfVzrYfbjn3UjESDRnEBB4GUDqwHTdU8TQaOrFSBO0H0CyKiutq4WMPpVaZ1LQEmNx0WmCHX9QxyWb2woKKG5jFIpt3Bp6UmHwUBCYYalK5zRg0pKEHk3VOecAUj6sq0qaoCjdesbLnY9pm7ozi0GmAFftkucDpRX9NZZvjNvV6qbmLyM9oC3b2bezfZlMV41smqr0W1vOvgG4BMfC5ZMvUDXOM1wRknyeOyOFSxTcMpSuljO2vUUjOxg7rYiy9BK6MjJnbwaKyO5JZ8MidPWpvMJb16iAv8FwpTJr8xYSdg3EdSfQBPCrC9LmyBXJDIqxa0V9Qcm9Ee2r1lmfIsYH2uagRkcIJ4P8Ub0nJfEbG2WGPwN8q9YnPWpV1sZ2F0Gh6VI7yzp2rQYZL6rXh8j4jiSLGl1vaxj7:10|c", "example", NULL, 10, METRIC_TYPE_COUNTER, SIGN_NONE);
    END_TEST();
}

#endif


* fix a bug (eval_appmacro)

* fix a bug (eval_appmacro)

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: eval.rl 2010-05-26 01:10:04 nineties $
 %);

include(stddef,code);
export(init_evaluator, assign, deref, check_arity, eval_sexp);

symbol_map: NULL; (% (symbol name, scope id) -> symbol object %);

scopeid: 0;
scopeid_stack: NULL;

(% p0: (symbol name, scope id) %);
symbol_hash: (p0) {
    return strhash(p0[0])*3 + p0[1];
};

symbol_equal: (p0, p1) {
    if (p0[1] != p1[1]) { return FALSE; };
    return streq(p0[0], p1[0]);
};

scope_push: () {
    scopeid = scopeid + 1;
    vec_pushback(scopeid_stack, scopeid);
};

scope_pop: () {
    vec_popback(scopeid_stack);
};

(% p0: symbol name, p1: value %);
assign: (p0, p1) {
    allocate(1);
    x0 = vec_at(scopeid_stack, vec_size(scopeid_stack)-1);
    map_add(symbol_map, mktup2(p0, x0), p1);
};

(% p0: symbol object %);
deref: (p0) {
    allocate(4);
    x0 = vec_size(scopeid_stack)-1;
    x3 = sym_name(p0);
    while (x0 >= 0) {
        x1 = vec_at(scopeid_stack, x0); (% scopeid-id %);
        x2 = map_find(symbol_map, mktup2(x3, x1));
        if (x2 != NULL) { return x2; };
        x0 = x0 - 1;
    };
    return p0;
};

check_arity: (p0, p1, p2) {
    if (length(p0) != p1) {
        fputs(stderr, "ERROR '");
        fputs(stderr, p2);
        fputs(stderr, "': required ");
        fputi(stderr, p1);
        fputs(stderr, " arguments");
        exit(1);
    }
};

eval_args: (p0) {
    if (p0 == nil_sym) { return nil_sym; };
    return mkcons(eval_sexp(car(p0)), eval_args(cdr(p0)));
};

eval_cons: (p0) {
    allocate(2);
    x0 = car(p0);
    if (sym_p(x0) == nil_sym) { goto &eval_cons_error; };
    x1 = eval_sexp(x0);
    if (x1 == var_sym)    { return eval_var(cdr(p0)); };
    if (x1 == set_sym)    { return eval_set(cdr(p0)); };
    if (x1 == quote_sym)  { return eval_quote(cdr(p0)); };
    if (x1 == if_sym)     { return eval_if(cdr(p0)); };
    if (x1 == cond_sym)   { return eval_cond(cdr(p0)); };
    if (x1 == while_sym)  { return eval_while(cdr(p0)); };
    if (x1 == do_sym)     { return eval_do(cdr(p0)); };
    if (x1 == lambda_sym) { return eval_lambda(cdr(p0)); };
    if (x1 == macro_sym)  { return eval_macro(cdr(p0)); };
    if (prim_p(x1) != nil_sym)   { return (prim_funptr(x1))(eval_args(cdr(p0))); };
    if (lambda_p(x1) != nil_sym) { return eval_applambda(x1, eval_args(cdr(p0))); };
    if (macro_p(x1) != nil_sym)  { return eval_appmacro(x1, cdr(p0)); };
label eval_cons_error;
    fputs(stderr, "ERROR: invalid application of '");
    pp_sexp(stderr, car(p0));
    fputs(stderr, "'\n");
    exit(1);
};

(% (var <id> <value>) %);
eval_var: (p0) {
    allocate(2);
    check_arity(p0, 2, "var");
    x0 = car(p0);
    x1 = eval_sexp(cadr(p0));
    sym_set(x0, x1);
    assign(sym_name(x0), x0);
    return x0;
};

eval_set: (p0) {
    allocate(2);
    check_arity(p0, 2, "set");
    x0 = deref(car(p0));
    x1 = eval_sexp(cadr(p0));
    if (sym_value(x0) == NULL) {
        fputs(stderr, "ERROR 'eval_set': undefined variable '");
        fputs(stderr, sym_name(x0));
        fputs(stderr, "'\n");
        exit(1);
    };
    sym_set(x0, x1);
    return x0;
};

eval_quote: (p0) {
    check_arity(p0, 1, "quote");
    return car(p0);
};

(% p0 : (cond ifthen ifelse) %);
eval_if: (p0) {
    allocate(2);
    check_arity(p0, 3, "if");
    scope_push();
    x0 = eval_sexp(car(p0));
    p0 = cdr(p0);
    if (x0 != nil_sym)  {
        x1 = eval_sexp(car(p0));
        scope_pop();
        return x1;
    };
    x1 = eval_sexp(cadr(p0));
    scope_pop();
    return x1;
};

eval_cond: (p0) {
    allocate(1);
    scope_push();
    while (p0 != nil_sym) {
        x0 = car(p0);
        check_arity(x0, 2, "cond");
        if (eval_sexp(car(x0)) != nil_sym) {
            scope_pop();
            return eval_sexp(cadr(x0));
        };
        p0 = cdr(p0);
    };
    fputs(stderr, "ERROR 'eval_cond': any condition was not met\n");
    exit(1);
};

(% p0: (cond body) %);
eval_while: (p0) {
    allocate(3);
    check_arity(p0, 2, "while");
    scope_push();
    x0 = car(p0); (% condition %);
    x1 = cadr(p0); (% body %);
    x2 = eval_sexp(x0);
    while (x2 != nil_sym) {
        eval_sexp(x1);
        x2 = eval_sexp(x0);
    };
    scope_pop();
    if (x2 != nil_sym) {
        fputs(stderr, "ERROR 'while': conditional expression could not evaluated to true/false\n");
        exit(1);
    };
    return nil_sym;
};

eval_do: (p0) {
    allocate(1);
    scope_push();
    while (p0 != nil_sym) {
        x0 = eval_sexp(car(p0));
        p0 = cdr(p0);
    };
    scope_pop();
    return x0;
};

(% (lambda (params) body) %);
eval_lambda: (p0) {
    allocate(2);
    check_arity(p0, 2, "lambda");
    x0 = car(p0); (% params %);
    x1 = cadr(p0); (% body %);
    return mklambda(x0, x1);
};

(% (macro (params) body) %);
eval_macro: (p0) {
    allocate(2);
    check_arity(p0, 2, "macro");
    x0 = car(p0); (% params %);
    x1 = cadr(p0); (% body %);
    return mkmacro(x0, x1);
};

(% p0: lambda, p1: params %);
eval_applambda: (p0, p1) {
    allocate(3);
    x0 = lambda_params(p0);
    if (length(x0) != length(p1)) {
        fputs(stderr, "ERROR: invalid number of argument (must be ");
        fputi(stderr, length(x0));
        fputi(stderr, ")\n");
        exit(1);
    };
    p1 = eval_args(p1);

    scope_push();
    while (x0 != nil_sym) {
        x1 = mksym(sym_name(car(x0)));
        sym_set(x1, car(p1));
        assign(sym_name(x1), x1);
        x0 = cdr(x0);
        p1 = cdr(p1);
    };
    x2 = eval_sexp(lambda_body(p0));
    scope_pop();
    return x2;
};

(% p0: macro, p1: params %);
eval_appmacro: (p0, p1) {
    allocate(3);
    x0 = macro_params(p0);
    if (length(x0) != length(p1)) {
        fputs(stderr, "ERROR: invalid number of argument (must be ");
        fputi(stderr, length(x0));
        fputi(stderr, ")\n");
        exit(1);
    };
    p1 = eval_args(p1);

    scope_push();
    while (x0 != nil_sym) {
        x1 = mksym(sym_name(car(x0)));
        sym_set(x1, car(p1));
        assign(sym_name(x1), x1);
        x0 = cdr(x0);
        p1 = cdr(p1);
    };
    x2 = eval_sexp(macro_body(p0));
    scope_pop();
    return eval_sexp(x2);
};

eval_sexp: (p0) {
    allocate(2);
    x0 = p0[0]; (% node code %);
    if (x0 == NODE_CONS)   { return eval_cons(p0); };
    if (x0 == NODE_SYMBOL) {
        x1 = deref(p0);
        if (sym_value(x1) != NULL) {
            return eval_sexp(sym_value(x1));
        };
        return x1;
    };
    return p0;
};

init_evaluator: () {
    symbol_map = mkmap(&symbol_hash, &symbol_equal, 100);
    scopeid_stack = mkvec(0);
    scope_push(); (% global scope %);
};

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: eval.rl 2010-05-25 22:10:05 nineties $
 %);

include(stddef,code);
export(init_evaluator, assign, deref, check_arity, eval_sexp);

symbol_map: NULL; (% (symbol name, scope id) -> symbol object %);

scopeid: 0;
scopeid_stack: NULL;

(% p0: (symbol name, scope id) %);
symbol_hash: (p0) {
    return strhash(p0[0])*3 + p0[1];
};

symbol_equal: (p0, p1) {
    if (p0[1] != p1[1]) { return FALSE; };
    return streq(p0[0], p1[0]);
};

scope_push: () {
    scopeid = scopeid + 1;
    vec_pushback(scopeid_stack, scopeid);
};

scope_pop: () {
    vec_popback(scopeid_stack);
};

(% p0: symbol name, p1: value %);
assign: (p0, p1) {
    allocate(1);
    x0 = vec_at(scopeid_stack, vec_size(scopeid_stack)-1);
    map_add(symbol_map, mktup2(p0, x0), p1);
};

(% p0: symbol object %);
deref: (p0) {
    allocate(4);
    x0 = vec_size(scopeid_stack)-1;
    x3 = sym_name(p0);
    while (x0 >= 0) {
        x1 = vec_at(scopeid_stack, x0); (% scopeid-id %);
        x2 = map_find(symbol_map, mktup2(x3, x1));
        if (x2 != NULL) { return x2; };
        x0 = x0 - 1;
    };
    return p0;
};

check_arity: (p0, p1, p2) {
    if (length(p0) != p1) {
        fputs(stderr, "ERROR '");
        fputs(stderr, p2);
        fputs(stderr, "': required ");
        fputi(stderr, p1);
        fputs(stderr, " arguments");
        exit(1);
    }
};

eval_args: (p0) {
    if (p0 == nil_sym) { return nil_sym; };
    return mkcons(eval_sexp(car(p0)), eval_args(cdr(p0)));
};

eval_cons: (p0) {
    allocate(2);
    x0 = car(p0);
    if (sym_p(x0) == nil_sym) { goto &eval_cons_error; };
    x1 = eval_sexp(x0);
    if (x1 == var_sym)    { return eval_var(cdr(p0)); };
    if (x1 == set_sym)    { return eval_set(cdr(p0)); };
    if (x1 == quote_sym)  { return eval_quote(cdr(p0)); };
    if (x1 == if_sym)     { return eval_if(cdr(p0)); };
    if (x1 == cond_sym)   { return eval_cond(cdr(p0)); };
    if (x1 == while_sym)  { return eval_while(cdr(p0)); };
    if (x1 == do_sym)     { return eval_do(cdr(p0)); };
    if (x1 == lambda_sym) { return eval_lambda(cdr(p0)); };
    if (x1 == macro_sym)  { return eval_macro(cdr(p0)); };
    if (prim_p(x1) != nil_sym)   { return (prim_funptr(x1))(eval_args(cdr(p0))); };
    if (lambda_p(x1) != nil_sym) { return eval_applambda(x1, eval_args(cdr(p0))); };
    if (macro_p(x1) != nil_sym)  { return eval_appmacro(x1, cdr(p0)); };
label eval_cons_error;
    fputs(stderr, "ERROR: invalid application of '");
    pp_sexp(stderr, car(p0));
    fputs(stderr, "'\n");
    exit(1);
};

(% (var <id> <value>) %);
eval_var: (p0) {
    allocate(2);
    check_arity(p0, 2, "var");
    x0 = car(p0);
    x1 = eval_sexp(cadr(p0));
    sym_set(x0, x1);
    assign(sym_name(x0), x0);
    return x0;
};

eval_set: (p0) {
    allocate(2);
    check_arity(p0, 2, "set");
    x0 = deref(car(p0));
    x1 = eval_sexp(cadr(p0));
    if (sym_value(x0) == NULL) {
        fputs(stderr, "ERROR 'eval_set': undefined variable '");
        fputs(stderr, sym_name(x0));
        fputs(stderr, "'\n");
        exit(1);
    };
    sym_set(x0, x1);
    return x0;
};

eval_quote: (p0) {
    check_arity(p0, 1, "quote");
    return car(p0);
};

(% p0 : (cond ifthen ifelse) %);
eval_if: (p0) {
    allocate(2);
    check_arity(p0, 3, "if");
    scope_push();
    x0 = eval_sexp(car(p0));
    p0 = cdr(p0);
    if (x0 != nil_sym)  {
        x1 = eval_sexp(car(p0));
        scope_pop();
        return x1;
    };
    x1 = eval_sexp(cadr(p0));
    scope_pop();
    return x1;
};

eval_cond: (p0) {
    allocate(1);
    scope_push();
    while (p0 != nil_sym) {
        x0 = car(p0);
        check_arity(x0, 2, "cond");
        if (eval_sexp(car(x0)) != nil_sym) {
            scope_pop();
            return eval_sexp(cadr(x0));
        };
        p0 = cdr(p0);
    };
    fputs(stderr, "ERROR 'eval_cond': any condition was not met\n");
    exit(1);
};

(% p0: (cond body) %);
eval_while: (p0) {
    allocate(3);
    check_arity(p0, 2, "while");
    scope_push();
    x0 = car(p0); (% condition %);
    x1 = cadr(p0); (% body %);
    x2 = eval_sexp(x0);
    while (x2 != nil_sym) {
        eval_sexp(x1);
        x2 = eval_sexp(x0);
    };
    scope_pop();
    if (x2 != nil_sym) {
        fputs(stderr, "ERROR 'while': conditional expression could not evaluated to true/false\n");
        exit(1);
    };
    return nil_sym;
};

eval_do: (p0) {
    allocate(1);
    scope_push();
    while (p0 != nil_sym) {
        x0 = eval_sexp(car(p0));
        p0 = cdr(p0);
    };
    scope_pop();
    return x0;
};

(% (lambda (params) body) %);
eval_lambda: (p0) {
    allocate(2);
    check_arity(p0, 2, "lambda");
    x0 = car(p0); (% params %);
    x1 = cadr(p0); (% body %);
    return mklambda(x0, x1);
};

(% (macro (params) body) %);
eval_macro: (p0) {
    allocate(2);
    check_arity(p0, 2, "macro");
    x0 = car(p0); (% params %);
    x1 = cadr(p0); (% body %);
    return mkmacro(x0, x1);
};

(% p0: lambda, p1: params %);
eval_applambda: (p0, p1) {
    allocate(3);
    x0 = lambda_params(p0);
    if (length(x0) != length(p1)) {
        fputs(stderr, "ERROR: invalid number of argument (must be ");
        fputi(stderr, length(x0));
        fputi(stderr, ")\n");
        exit(1);
    };
    p1 = eval_args(p1);

    scope_push();
    while (x0 != nil_sym) {
        x1 = mksym(sym_name(car(x0)));
        sym_set(x1, car(p1));
        assign(sym_name(x1), x1);
        x0 = cdr(x0);
        p1 = cdr(p1);
    };
    x2 = eval_sexp(lambda_body(p0));
    scope_pop();
    return x2;
};

(% p0: macro, p1: params %);
eval_appmacro: (p0, p1) {
    allocate(3);
    x0 = macro_params(p0);
    if (length(x0) != length(p1)) {
        fputs(stderr, "ERROR: invalid number of argument (must be ");
        fputi(stderr, length(x0));
        fputi(stderr, ")\n");
        exit(1);
    };
    p1 = eval_args(p1);

    scope_push();
    while (x0 != nil_sym) {
        x1 = mksym(sym_name(car(x0)));
        sym_set(x1, car(p1));
        assign(sym_name(x1), x1);
        x0 = cdr(x0);
        p1 = cdr(p1);
    };
    x2 = eval_sexp(macro_body(p0));
    scope_pop();
    return x2;
};

eval_sexp: (p0) {
    allocate(2);
    x0 = p0[0]; (% node code %);
    if (x0 == NODE_CONS)   { return eval_cons(p0); };
    if (x0 == NODE_SYMBOL) {
        x1 = deref(p0);
        if (sym_value(x1) != NULL) {
            return eval_sexp(sym_value(x1));
        };
        return x1;
    };
    return p0;
};

init_evaluator: () {
    symbol_map = mkmap(&symbol_hash, &symbol_equal, 100);
    scopeid_stack = mkvec(0);
    scope_push(); (% global scope %);
};


formatting

formatting

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

int calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  size_t j;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0)

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_NESTED && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("nested qstructs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].nested_name = curr_item.nested_name;
      def->items[curr_item_index].nested_name_len = curr_item.nested_name_len;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    lc_alpha_u = [a-z] | '_';
    uc_alpha_u = [A-Z] | '_';
    identifier = lc_alpha_u $!{ PARSE_ERROR("qstruct item names must start with lowercase letters"); }
                 alnum_u*;
    identifier_with_package = uc_alpha_u $!{ PARSE_ERROR("qstruct names must start with uppercase letters"); }
                              alnum_u* ('::' uc_alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           identifier_with_package >{ curr_item.nested_name = p; curr_item.type = QSTRUCT_TYPE_NESTED; }
                                   %{ curr_item.nested_name_len = p - curr_item.nested_name; }
      ;

    array_spec = '['
                    ws*
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                    ws*
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 ;

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec?
             $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); }
      ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?
      ;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  // reverse def list
  for (curr_def=NULL; def;) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }
  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    for(j=0; j<def->num_items; j++) {
      if ((def->items[j].type & 0xFFFF) == QSTRUCT_TYPE_NESTED) {
        HASH_FIND(hh, def_hash_by_name, def->items[j].nested_name, def->items[j].nested_name_len, def_lookup);
        if (!def_lookup)
          PARSE_ERROR("type '%.*s' referred to before being defined", (int) def->items[j].nested_name_len, def->items[j].nested_name);

        def->items[j].nested_def = def_lookup;
      }
    }

    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    if (calculate_qstruct_packing(curr_def) < 0)
      PARSE_ERROR("memory error in packing");
  }

  bail:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

int calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  size_t j;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0)

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_NESTED && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("nested qstructs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].nested_name = curr_item.nested_name;
      def->items[curr_item_index].nested_name_len = curr_item.nested_name_len;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    lc_alpha_u = [a-z] | '_';
    uc_alpha_u = [A-Z] | '_';
    identifier = lc_alpha_u $!{ PARSE_ERROR("qstruct item names must start with lowercase letters"); }
                 alnum_u*;
    identifier_with_package = uc_alpha_u $!{ PARSE_ERROR("qstruct names must start with uppercase letters"); }
                              alnum_u* ('::' uc_alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           identifier_with_package >{ curr_item.nested_name = p; curr_item.type = QSTRUCT_TYPE_NESTED; }
                                   %{ curr_item.nested_name_len = p - curr_item.nested_name; }
      ;

    array_spec = ('['
                    ws*
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                    ws*
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec?
             $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); } ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  // reverse def list
  for (curr_def=NULL; def;) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }
  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    for(j=0; j<def->num_items; j++) {
      if ((def->items[j].type & 0xFFFF) == QSTRUCT_TYPE_NESTED) {
        HASH_FIND(hh, def_hash_by_name, def->items[j].nested_name, def->items[j].nested_name_len, def_lookup);
        if (!def_lookup)
          PARSE_ERROR("type '%.*s' referred to before being defined", (int) def->items[j].nested_name_len, def->items[j].nested_name);

        def->items[j].nested_def = def_lookup;
      }
    }

    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    if (calculate_qstruct_packing(curr_def) < 0)
      PARSE_ERROR("memory error in packing");
  }

  bail:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}


tag matching rules improvement

tag matching rules improvement

%%{
  machine feature_common;

  FEATURE = 'Feature:' >start_keyword %end_keyword;
  SCENARIO = 'Scenario:' >start_keyword %end_keyword;
  STEP = ('Given' | 'When' | 'And' | 'Then' | 'But') >start_keyword %end_keyword;

  EOL = ('\r'? '\n') @inc_line_number;
  Comment = space* '#' %begin_content ^EOL+ %store_comment_content EOL+;

  Feature_end = EOL+ space* ('Scenario:' | '@' | '#');

  Feature = space* FEATURE %begin_content %current_line ^Feature_end+ %/store_feature_content :>> Feature_end >backup @store_feature_content;
  Scenario = space* SCENARIO %begin_content %current_line ^EOL+ %store_scenario_content EOL+;  #SINGLE LINE ONLY
  Step = space* STEP %begin_content %current_line ^EOL+ %store_step_content EOL+;

  Tag = ( '@' [^@\r\n\t ]+ ) >begin_content %store_tag_content;
  Tags = space* (Tag @current_line space*)+ EOL+;

  feature = (
    start: (
      Comment ->start |
      Tags ->start |
      Feature ->feature_found
    ),
    feature_found: (
      Comment ->feature_found |
      Tags ->feature_found |
      Scenario ->scenario_found
    ),
    scenario_found: (
      Comment ->scenario_found |
      Tags ->feature_found |
      Scenario ->scenario_found |
      Step ->scenario_found
    )
  );

  main := feature;
}%%

%%{
  machine feature_common;

  FEATURE = 'Feature:' >start_keyword %end_keyword;
  SCENARIO = 'Scenario:' >start_keyword %end_keyword;
  STEP = ('Given' | 'When' | 'And' | 'Then' | 'But') >start_keyword %end_keyword;

  EOL = ('\r'? '\n') @inc_line_number;
  Comment = space* '#' %begin_content ^EOL+ %store_comment_content EOL+;

  Feature_end = EOL+ <: space* ('Scenario:' | '@' | '#');

  Feature = space* FEATURE %begin_content %current_line ^Feature_end+ %/store_feature_content :>> Feature_end >backup @store_feature_content;
  Scenario = space* SCENARIO %begin_content %current_line ^EOL+ %store_scenario_content EOL+;  #SINGLE LINE ONLY
  Step = space* STEP %begin_content %current_line ^EOL+ %store_step_content EOL+;

  Tag = ( '@' [^@\r\n\t ]+ ) >begin_content %store_tag_content;
  Tags = space* %current_line (Tag space*)+ EOL+;

  feature = (
    start: (
      Comment ->start |
      Tags ->start |
      Feature ->feature_found
    ),
    feature_found: (
      Comment ->feature_found |
      Tags ->feature_found |
      Scenario ->scenario_found
    ),
    scenario_found: (
      Comment ->scenario_found |
      Tags ->feature_found |
      Scenario ->scenario_found |
      Step ->scenario_found
    )
  );

  main := feature;
}%%