updates for D2 in cppscan5

updates for D2 in cppscan5

/*
 * @LANG: d
 */

/*
 * Test in and out state actions.
 */

import std.c.stdio;
import std.string;

static const int TK_Dlit = 192;
static const int TK_Slit = 193;
static const int TK_Float = 194;
static const int TK_Id = 195;
static const int TK_NameSep = 197;
static const int TK_Arrow = 211;
static const int TK_PlusPlus = 212;
static const int TK_MinusMinus = 213;
static const int TK_ArrowStar = 214;
static const int TK_DotStar = 215;
static const int TK_ShiftLeft = 216;
static const int TK_ShiftRight = 217;
static const int TK_IntegerDecimal = 218;
static const int TK_IntegerOctal = 219;
static const int TK_IntegerHex = 220;
static const int TK_EqualsEquals = 223;
static const int TK_NotEquals = 224;
static const int TK_AndAnd = 225;
static const int TK_OrOr = 226;
static const int TK_MultAssign = 227;
static const int TK_DivAssign = 228;
static const int TK_PercentAssign = 229;
static const int TK_PlusAssign = 230;
static const int TK_MinusAssign = 231;
static const int TK_AmpAssign = 232;
static const int TK_CaretAssign = 233;
static const int TK_BarAssign = 234;
static const int TK_DotDotDot = 240;
static const int TK_Whitespace = 241;
static const int TK_Comment = 242;

class Scanner
{
	int cs, act;
	const(char) *ts, te;

	void token( int tok )
	{
		const(char) *data = ts;
		long len = te - ts;
		printf( "<%i> ", tok );
		for ( int i = 0; i < len; i++ )
			printf( "%c", data[i] );
		printf( "\n" );
	}

	%%{

	machine Scanner;

	main := |*

	# Single and double literals.
	( 'L'? "'" ( [^'\\\n] | /\\./ )* "'" )
		=> { token( TK_Slit );};
	( 'L'? '"' ( [^"\\\n] | /\\./ )* '"' )
		=> { token( TK_Dlit );};

	# Identifiers
	( [a-zA-Z_] [a-zA-Z0-9_]* )
		=>{ token( TK_Id );};

	# Floating literals.
	fract_const = digit* '.' digit+ | digit+ '.';
	exponent = [eE] [+\-]? digit+;
	float_suffix = [flFL];

	( fract_const exponent? float_suffix? |
		digit+ exponent float_suffix? )
		=> { token( TK_Float );};

	# Integer decimal. Leading part buffered by float.
	( ( '0' | [1-9] [0-9]* ) [ulUL]{0,3} )
		=> { token( TK_IntegerDecimal );};

	# Integer octal. Leading part buffered by float.
	( '0' [0-9]+ [ulUL]{0,2} )
		=> { token( TK_IntegerOctal );};

	# Integer hex. Leading 0 buffered by float.
	( '0' ( 'x' [0-9a-fA-F]+ [ulUL]{0,2} ) )
		=> { token( TK_IntegerHex );};

	# Only buffer the second item, first buffered by symbol. */
	'::' => {token( TK_NameSep );};
	'==' => {token( TK_EqualsEquals );};
	'!=' => {token( TK_NotEquals );};
	'&&' => {token( TK_AndAnd );};
	'||' => {token( TK_OrOr );};
	'*=' => {token( TK_MultAssign );};
	'/=' => {token( TK_DivAssign );};
	'%=' => {token( TK_PercentAssign );};
	'+=' => {token( TK_PlusAssign );};
	'-=' => {token( TK_MinusAssign );};
	'&=' => {token( TK_AmpAssign );};
	'^=' => {token( TK_CaretAssign );};
	'|=' => {token( TK_BarAssign );};
	'++' => {token( TK_PlusPlus );};
	'--' => {token( TK_MinusMinus );};
	'->' => {token( TK_Arrow );};
	'->*' => {token( TK_ArrowStar );};
	'.*' => {token( TK_DotStar );};

	# Three char compounds, first item already buffered. */
	'...' => { token( TK_DotDotDot );};

	# Single char symbols.
	( punct - [_"'] ) => { token( ts[0] );};

	action comment {
		token( TK_Comment );
	}

	# Comments and whitespace.
	'/*' ( any* $0 '*/' @1 ) => comment;
	'//' ( any* $0 '\n' @1 ) => comment;
	( any - 33..126 )+ => { token( TK_Whitespace );};

	*|;

	}%%

	%% write data noprefix;

	void init( )
	{
		%% write init;
	}

	void execute( const(char)* data, ulong len )
	{
		const(char) *p = data;
		const(char) *pe = data + len;
		const(char) *eof = pe;

		%% write exec;
	}

	// Indicate that there is no more data. Returns -1 if the machine finishes
	// in the error state and does not accept, 0 if the machine finishes
	// in any other non-accepting state and 1 if the machine finishes in an
	// accepting state.
	int finish( )
	{
		if ( cs == error )
			return -1;
		if ( cs >= first_final )
			return 1;
		return 0;
	}
};

static const int BUFSIZE = 12;

void test( const(char) buf[] )
{
	Scanner scanner = new Scanner();
	scanner.init();

	scanner.execute( buf.ptr, buf.length );
	if ( scanner.cs == Scanner.error ) {
		/* Machine failed before finding a token. */
		printf("PARSE ERROR\n");
	}
	scanner.finish();
}

int main()
{
	test(
		"\"\\\"hi\" /*\n"
		"*/\n"
		"44 .44\n"
		"44. 44\n"
		"44 . 44\n"
		"44.44\n"
		"_hithere22"
	);

	test(
		"'\\''\"\\n\\d'\\\"\"\n"
		"hi\n"
		"99\n"
		".99\n"
		"99e-4\n"
		"->*\n"
		"||\n"
		"0x98\n"
		"0x\n"
		"//\n"
		"/* * */"
	);

	test(
		"'\n"
		"'\n"
	);

	return 0;
}

/+ _____OUTPUT_____
<192> "\"hi"
<241>
<242> /*
*/
<241>

<218> 44
<241>
<194> .44
<241>

<194> 44.
<241>
<218> 44
<241>

<218> 44
<241>
<46> .
<241>
<218> 44
<241>

<194> 44.44
<241>

<195> _hithere22
<193> '\''
<192> "\n\d'\""
<241>

<195> hi
<241>

<218> 99
<241>

<194> .99
<241>

<194> 99e-4
<241>

<214> ->*
<241>

<226> ||
<241>

<220> 0x98
<241>

<218> 0
<195> x
<241>

<242> //

<242> /* * */
PARSE ERROR
+++++++++++++++++++/

/*
 * @LANG: d
 */

/*
 * Test in and out state actions.
 */

import std.c.stdio;
import std.string;

static const int TK_Dlit = 192;
static const int TK_Slit = 193;
static const int TK_Float = 194;
static const int TK_Id = 195;
static const int TK_NameSep = 197;
static const int TK_Arrow = 211;
static const int TK_PlusPlus = 212;
static const int TK_MinusMinus = 213;
static const int TK_ArrowStar = 214;
static const int TK_DotStar = 215;
static const int TK_ShiftLeft = 216;
static const int TK_ShiftRight = 217;
static const int TK_IntegerDecimal = 218;
static const int TK_IntegerOctal = 219;
static const int TK_IntegerHex = 220;
static const int TK_EqualsEquals = 223;
static const int TK_NotEquals = 224;
static const int TK_AndAnd = 225;
static const int TK_OrOr = 226;
static const int TK_MultAssign = 227;
static const int TK_DivAssign = 228;
static const int TK_PercentAssign = 229;
static const int TK_PlusAssign = 230;
static const int TK_MinusAssign = 231;
static const int TK_AmpAssign = 232;
static const int TK_CaretAssign = 233;
static const int TK_BarAssign = 234;
static const int TK_DotDotDot = 240;
static const int TK_Whitespace = 241;
static const int TK_Comment = 242;

class Scanner
{
	int cs, act;
	char *ts, te;

	void token( int tok )
	{
		char *data = ts;
		int len = te - ts;
		printf( "<%i> ", tok );
		for ( int i = 0; i < len; i++ )
			printf( "%c", data[i] );
		printf( "\n" );
	}

	%%{

	machine Scanner;

	main := |*

	# Single and double literals.
	( 'L'? "'" ( [^'\\\n] | /\\./ )* "'" )
		=> { token( TK_Slit );};
	( 'L'? '"' ( [^"\\\n] | /\\./ )* '"' )
		=> { token( TK_Dlit );};

	# Identifiers
	( [a-zA-Z_] [a-zA-Z0-9_]* )
		=>{ token( TK_Id );};

	# Floating literals.
	fract_const = digit* '.' digit+ | digit+ '.';
	exponent = [eE] [+\-]? digit+;
	float_suffix = [flFL];

	( fract_const exponent? float_suffix? |
		digit+ exponent float_suffix? )
		=> { token( TK_Float );};

	# Integer decimal. Leading part buffered by float.
	( ( '0' | [1-9] [0-9]* ) [ulUL]{0,3} )
		=> { token( TK_IntegerDecimal );};

	# Integer octal. Leading part buffered by float.
	( '0' [0-9]+ [ulUL]{0,2} )
		=> { token( TK_IntegerOctal );};

	# Integer hex. Leading 0 buffered by float.
	( '0' ( 'x' [0-9a-fA-F]+ [ulUL]{0,2} ) )
		=> { token( TK_IntegerHex );};

	# Only buffer the second item, first buffered by symbol. */
	'::' => {token( TK_NameSep );};
	'==' => {token( TK_EqualsEquals );};
	'!=' => {token( TK_NotEquals );};
	'&&' => {token( TK_AndAnd );};
	'||' => {token( TK_OrOr );};
	'*=' => {token( TK_MultAssign );};
	'/=' => {token( TK_DivAssign );};
	'%=' => {token( TK_PercentAssign );};
	'+=' => {token( TK_PlusAssign );};
	'-=' => {token( TK_MinusAssign );};
	'&=' => {token( TK_AmpAssign );};
	'^=' => {token( TK_CaretAssign );};
	'|=' => {token( TK_BarAssign );};
	'++' => {token( TK_PlusPlus );};
	'--' => {token( TK_MinusMinus );};
	'->' => {token( TK_Arrow );};
	'->*' => {token( TK_ArrowStar );};
	'.*' => {token( TK_DotStar );};

	# Three char compounds, first item already buffered. */
	'...' => { token( TK_DotDotDot );};

	# Single char symbols.
	( punct - [_"'] ) => { token( ts[0] );};

	action comment {
		token( TK_Comment );
	}

	# Comments and whitespace.
	'/*' ( any* $0 '*/' @1 ) => comment;
	'//' ( any* $0 '\n' @1 ) => comment;
	( any - 33..126 )+ => { token( TK_Whitespace );};

	*|;

	}%%

	%% write data noprefix;

	void init( )
	{
		%% write init;
	}

	void execute( char* data, int len )
	{
		char *p = data;
		char *pe = data + len;
		char *eof = pe;

		%% write exec;
	}

	// Indicate that there is no more data. Returns -1 if the machine finishes
	// in the error state and does not accept, 0 if the machine finishes
	// in any other non-accepting state and 1 if the machine finishes in an
	// accepting state.
	int finish( )
	{
		if ( cs == error )
			return -1;
		if ( cs >= first_final )
			return 1;
		return 0;
	}
};

static const int BUFSIZE = 12;

void test( char buf[] )
{
	Scanner scanner = new Scanner();
	scanner.init();

	scanner.execute( buf.ptr, buf.length );
	if ( scanner.cs == Scanner.error ) {
		/* Machine failed before finding a token. */
		printf("PARSE ERROR\n");
	}
	scanner.finish();
	return 0;
}

int main()
{
	test(
		"\"\\\"hi\" /*\n"
		"*/\n"
		"44 .44\n"
		"44. 44\n"
		"44 . 44\n"
		"44.44\n"
		"_hithere22"
	);

	test(
		"'\\''\"\\n\\d'\\\"\"\n"
		"hi\n"
		"99\n"
		".99\n"
		"99e-4\n"
		"->*\n"
		"||\n"
		"0x98\n"
		"0x\n"
		"//\n"
		"/* * */"
	);

	test(
		"'\n"
		"'\n"
	);

	return 0;
}

/+ _____OUTPUT_____
<192> "\"hi"
<241>
<242> /*
*/
<241>

<218> 44
<241>
<194> .44
<241>

<194> 44.
<241>
<218> 44
<241>

<218> 44
<241>
<46> .
<241>
<218> 44
<241>

<194> 44.44
<241>

<195> _hithere22
<193> '\''
<192> "\n\d'\""
<241>

<195> hi
<241>

<218> 99
<241>

<194> .99
<241>

<194> 99e-4
<241>

<214> ->*
<241>

<226> ||
<241>

<220> 0x98
<241>

<218> 0
<195> x
<241>

<242> //

<242> /* * */
PARSE ERROR
+++++++++++++++++++/


fix dangling <li>

fix dangling <li>

double_return now matches double or more returns

perhaps the name: 'double_return' should be changed also

http://jgarber.lighthouseapp.com/projects/13054/tickets/183-three-newlines-after-a-list-at-end-of-file-make-dangling-li

/*
 * redcloth_scan.rl
 *
 * Copyright (C) 2009 Jason Garber
 */
%%{

  machine redcloth_scan;

  # blocks
  notextile_tag = notextile (LF+ | EOF) ;
  noparagraph_line_start = " "+ ;
  notextile_block_start = ( "notextile" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " "+ ) %SET_ATTR ;
  pre_tag_start = "<pre" [^>]* ">" (space* code_tag_start)? ;
  pre_tag_end = (code_tag_end space*)? "</pre>" LF? ;
  pre_block_start = ( "pre" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " " ) %SET_ATTR ;
  bc_start = ( "bc" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " " ) %SET_ATTR ;
  bq_start = ( "bq" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) ( ":" %A uri %{ STORE("cite"); } )? " "+ ) %SET_ATTR ;
  non_ac_btype = ( "bq" | "bc" | "pre" | "notextile" | "table" );
  btype = (alpha alnum*) -- (non_ac_btype | "fn" digit+);
  block_start = ( btype >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " "+ ) >B %{ STORE_B("fallback"); } %SET_ATTR ;
  all_btypes = btype | non_ac_btype;
  next_block_start = ( all_btypes A_noactions C_noactions :> "."+ " " ) >A @{ fexec(reg); } ;
  double_return = LF [ \t]* LF LF* ;
  block_end = ( double_return | EOF );
  ftype = ( "fn" >A %{ STORE("type"); } digit+ >A %{ STORE("id"); } ) ;
  footnote_start = ( ftype A C :> dotspace ) %SET_ATTR ;
  ul = "*" %{NEST(); SET_LIST_TYPE("ul");};
  ol = "#" %{NEST(); SET_LIST_TYPE("ol");};
  ul_start  = ( ul | ol )* ul A_HLGN_noactions* C_noactions :> " "+ ;
  ol_start  = ( ul | ol )* ol N A_HLGN_noactions* C_noactions :> " "+ ;
  list_start  = " "* A_HLGN* C ( ul_start | ol_start ) >B >{RESET_NEST();} @{ fexec(bck); } ;

  dt_start = "-" . " "+ ;
  dd_start = ":=" ;
  long_dd  = dd_start " "* LF %{ ADD_BLOCK(); ASET("type", "dd"); } any+ >A %{ TRANSFORM("text"); } :>> "=:" ;
  dl_start = (dt_start mtext (LF dt_start mtext)* " "* dd_start)  ;
  blank_line = LF;
  link_alias = ( "[" >{ ASET("type", "ignore"); } %A chars %T "]" %A uri %{ STORE_URL("href"); } ) ;
  horizontal_rule = '*'{3,} | '-'{3,} | '_'{3,} ;

  # image lookahead
  IMG_A_LEFT = "<" %{ ASET("float", "left"); } ;
  IMG_A_RIGHT = ">" %{ ASET("float", "right"); } ;
  aligned_image = ( "["? "!" (IMG_A_LEFT | IMG_A_RIGHT) ) >A @{ fexec(reg); } ;

  # html blocks
  BlockTagName = Name - ("pre" | "notextile" | "a" | "applet" | "basefont" | "bdo" | "br" | "font" | "iframe" | "img" | "map" | "object" | "param" | "embed" | "q" | "script" | "span" | "sub" | "sup" | "abbr" | "acronym" | "cite" | "code" | "del" | "dfn" | "em" | "ins" | "kbd" | "samp" | "strong" | "var" | "b" | "big" | "i" | "s" | "small" | "strike" | "tt" | "u");
  block_start_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? ">" | "<" BlockTagName ">";
  block_empty_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? "/>" | "<" BlockTagName "/>" ;
  block_end_tag = "</" BlockTagName space* ">" ;
  html_start = indent >B %{STORE_B("indent_before_start");} block_start_tag >B %{STORE_B("start_tag");}  indent >B %{STORE_B("indent_after_start");} ;
  html_end = indent >B %{STORE_B("indent_before_end");} block_end_tag >B %{STORE_B("end_tag");} (indent LF?) >B %{STORE_B("indent_after_end");} ;
  standalone_html = indent (block_start_tag | block_empty_tag | block_end_tag) indent (LF+ | EOF);
  html_end_terminating_block = ( LF indent block_end_tag ) >A @{ fexec(reg); } ;

  # tables
  td_text = (default - LF)+ ;
  btext = (td_text (LF td_text)* ( LF{2} )?) | [\t ]* ;
  tddef = ( D? S A C :> dotspace ) %SET_ATTR ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, "text", "td");} ) >X ;
  trdef = ( A C :> dotspace ) %SET_ATTR ;
  tr = ( trdef? "|" %{INLINE(table, "tr_open");} td+ ) >X %{INLINE(table, "tr_close");} ;
  trows = ( tr (LF >X tr)* ) ;
  tdef = ( "table" %X A C :> dotspace LF ) %SET_ATTR ;
  table = ( tdef? trows >{CLEAR(table); INLINE(table, "table_open"); RESET_REG();} ) ;

  # info
  redcloth_version = ("RedCloth" >A ("::" | " " ) "VERSION"i ":"? " ")? %{STORE("prefix");} "RedCloth::VERSION" (LF* EOF | double_return) ;

  pre_tag := |*
    pre_tag_end         { CAT(block); DONE(block); fgoto main; };
    default => esc_pre;
  *|;

  pre_block := |*
    EOF { ADD_BLOCKCODE(); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCKCODE(); };
    double_return when not_extended { ADD_BLOCKCODE(); fgoto main; } ;
    double_return next_block_start when extended { ADD_EXTENDED_BLOCKCODE(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCKCODE(); fgoto main; };
    default => esc_pre;
  *|;

  script_tag := |*
    script_tag_end   { CAT(block); ASET("type", "ignore"); ADD_BLOCK(); fgoto main; };
    EOF              { ASET("type", "ignore"); ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  noparagraph_line := |*
    LF  { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  notextile_block := |*
    EOF { ADD_BLOCK(); fgoto main; };
    double_return when extended { CAT(block); ADD_EXTENDED_BLOCK(); CAT(html); };
    double_return when not_extended { ADD_BLOCK(); CAT(html); fgoto main; } ;
    double_return next_block_start when extended { CAT(block); ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); CAT(html); fgoto main; };
    default => cat;
  *|;

  html := |*
    html_end        { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  bc := |*
    EOF {  ADD_BLOCKCODE();  INLINE(html, "bc_close");  SET_PLAIN_BLOCK("p"); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCKCODE(); CAT(html); };
    double_return when not_extended { ADD_BLOCKCODE(); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCKCODE(); CAT(html); RSTRIP_BANG(html); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCKCODE(); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); fgoto main; };
    default => esc_pre;
  *|;

  bq := |*
    EOF { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCK(); };
    double_return when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCK(); INLINE(html, "bq_close"); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    html_end_terminating_block when extended { ADD_EXTENDED_BLOCK(); INLINE(html, "bq_close"); END_EXTENDED(); fgoto main; };
    html_end_terminating_block when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    default => cat;
  *|;

  block := |*
    EOF { ADD_BLOCK(); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCK(); };
    double_return when not_extended { ADD_BLOCK(); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); fgoto main; };
    html_end_terminating_block when extended { ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    html_end_terminating_block when not_extended { ADD_BLOCK(); fgoto main; };
    LF list_start { ADD_BLOCK(); CLEAR_LIST(); LIST_LAYOUT(); fgoto list_item; };

    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  ul_item  = ( ul | ol )* ul A_HLGN* C :> " "+ ;
  ol_item  = ( ul | ol )* ol N_noactions A_HLGN* C :> " "+ ;
  list_item  := (" "* ( ul_item | ol_item )) @{ SET_ATTRIBUTES(); fgoto list_content; } ;

  list_content := |*
    LF list_start { ADD_BLOCK(); LIST_LAYOUT(); fgoto list_item; };
    block_end     { ADD_BLOCK(); RESET_NEST(); LIST_LAYOUT(); fgoto main; };
    default => cat;
  *|;

  dl := |*
    LF dt_start     { ADD_BLOCK(); ASET("type", "dt"); };
    dd_start        { ADD_BLOCK(); ASET("type", "dd"); };
    long_dd         { INLINE(html, "dd"); CLEAR_REGS(); };
    block_end       { ADD_BLOCK(); INLINE(html, "dl_close");  fgoto main; };
    default => cat;
  *|;

  main := |*
    noparagraph_line_start  { ASET("type", "ignored_line"); fgoto noparagraph_line; };
    notextile_tag   { INLINE(html, "notextile"); };
    notextile_block_start { ASET("type", "notextile"); fgoto notextile_block; };
    script_tag_start { CAT(block); fgoto script_tag; };
    pre_tag_start       { ASET("type", "notextile"); CAT(block); fgoto pre_tag; };
    pre_block_start { fgoto pre_block; };
    standalone_html { ASET("type", "html"); CAT(block); ADD_BLOCK(); };
    html_start      { ASET("type", "html_block"); fgoto html; };
    bc_start        { INLINE(html, "bc_open"); ASET("type", "code"); SET_PLAIN_BLOCK("code"); fgoto bc; };
    bq_start        { INLINE(html, "bq_open"); ASET("type", "p"); fgoto bq; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    horizontal_rule { INLINE(html, "hr"); };
    list_start      { CLEAR_LIST(); LIST_LAYOUT(); fgoto list_item; };
    dl_start        { fexec(ts + 1); INLINE(html, "dl_open"); ASET("type", "dt"); fgoto dl; };
    table           { INLINE(table, "table_close"); DONE(table); fgoto block; };
    link_alias      { STORE_LINK_ALIAS(); DONE(block); };
    aligned_image   { RESET_TYPE(); fgoto block; };
    redcloth_version { INLINE(html, "redcloth_version"); };
    blank_line => cat;
    default
    {
      CLEAR_REGS();
      RESET_TYPE();
      CAT(block);
      fgoto block;
    };
    EOF;
  *|;

}%%;

/*
 * redcloth_scan.rl
 *
 * Copyright (C) 2009 Jason Garber
 */
%%{

  machine redcloth_scan;

  # blocks
  notextile_tag = notextile (LF+ | EOF) ;
  noparagraph_line_start = " "+ ;
  notextile_block_start = ( "notextile" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " "+ ) %SET_ATTR ;
  pre_tag_start = "<pre" [^>]* ">" (space* code_tag_start)? ;
  pre_tag_end = (code_tag_end space*)? "</pre>" LF? ;
  pre_block_start = ( "pre" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " " ) %SET_ATTR ;
  bc_start = ( "bc" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " " ) %SET_ATTR ;
  bq_start = ( "bq" >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) ( ":" %A uri %{ STORE("cite"); } )? " "+ ) %SET_ATTR ;
  non_ac_btype = ( "bq" | "bc" | "pre" | "notextile" | "table" );
  btype = (alpha alnum*) -- (non_ac_btype | "fn" digit+);
  block_start = ( btype >A %{ STORE("type"); } A C :> "." ( "." %extend | "" ) " "+ ) >B %{ STORE_B("fallback"); } %SET_ATTR ;
  all_btypes = btype | non_ac_btype;
  next_block_start = ( all_btypes A_noactions C_noactions :> "."+ " " ) >A @{ fexec(reg); } ;
  double_return = LF [ \t]* LF ;
  block_end = ( double_return | EOF );
  ftype = ( "fn" >A %{ STORE("type"); } digit+ >A %{ STORE("id"); } ) ;
  footnote_start = ( ftype A C :> dotspace ) %SET_ATTR ;
  ul = "*" %{NEST(); SET_LIST_TYPE("ul");};
  ol = "#" %{NEST(); SET_LIST_TYPE("ol");};
  ul_start  = ( ul | ol )* ul A_HLGN_noactions* C_noactions :> " "+ ;
  ol_start  = ( ul | ol )* ol N A_HLGN_noactions* C_noactions :> " "+ ;
  list_start  = " "* A_HLGN* C ( ul_start | ol_start ) >B >{RESET_NEST();} @{ fexec(bck); } ;

  dt_start = "-" . " "+ ;
  dd_start = ":=" ;
  long_dd  = dd_start " "* LF %{ ADD_BLOCK(); ASET("type", "dd"); } any+ >A %{ TRANSFORM("text"); } :>> "=:" ;
  dl_start = (dt_start mtext (LF dt_start mtext)* " "* dd_start)  ;
  blank_line = LF;
  link_alias = ( "[" >{ ASET("type", "ignore"); } %A chars %T "]" %A uri %{ STORE_URL("href"); } ) ;
  horizontal_rule = '*'{3,} | '-'{3,} | '_'{3,} ;

  # image lookahead
  IMG_A_LEFT = "<" %{ ASET("float", "left"); } ;
  IMG_A_RIGHT = ">" %{ ASET("float", "right"); } ;
  aligned_image = ( "["? "!" (IMG_A_LEFT | IMG_A_RIGHT) ) >A @{ fexec(reg); } ;

  # html blocks
  BlockTagName = Name - ("pre" | "notextile" | "a" | "applet" | "basefont" | "bdo" | "br" | "font" | "iframe" | "img" | "map" | "object" | "param" | "embed" | "q" | "script" | "span" | "sub" | "sup" | "abbr" | "acronym" | "cite" | "code" | "del" | "dfn" | "em" | "ins" | "kbd" | "samp" | "strong" | "var" | "b" | "big" | "i" | "s" | "small" | "strike" | "tt" | "u");
  block_start_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? ">" | "<" BlockTagName ">";
  block_empty_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? "/>" | "<" BlockTagName "/>" ;
  block_end_tag = "</" BlockTagName space* ">" ;
  html_start = indent >B %{STORE_B("indent_before_start");} block_start_tag >B %{STORE_B("start_tag");}  indent >B %{STORE_B("indent_after_start");} ;
  html_end = indent >B %{STORE_B("indent_before_end");} block_end_tag >B %{STORE_B("end_tag");} (indent LF?) >B %{STORE_B("indent_after_end");} ;
  standalone_html = indent (block_start_tag | block_empty_tag | block_end_tag) indent (LF+ | EOF);
  html_end_terminating_block = ( LF indent block_end_tag ) >A @{ fexec(reg); } ;

  # tables
  td_text = (default - LF)+ ;
  btext = (td_text (LF td_text)* ( LF{2} )?) | [\t ]* ;
  tddef = ( D? S A C :> dotspace ) %SET_ATTR ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, "text", "td");} ) >X ;
  trdef = ( A C :> dotspace ) %SET_ATTR ;
  tr = ( trdef? "|" %{INLINE(table, "tr_open");} td+ ) >X %{INLINE(table, "tr_close");} ;
  trows = ( tr (LF >X tr)* ) ;
  tdef = ( "table" %X A C :> dotspace LF ) %SET_ATTR ;
  table = ( tdef? trows >{CLEAR(table); INLINE(table, "table_open"); RESET_REG();} ) ;

  # info
  redcloth_version = ("RedCloth" >A ("::" | " " ) "VERSION"i ":"? " ")? %{STORE("prefix");} "RedCloth::VERSION" (LF* EOF | double_return) ;

  pre_tag := |*
    pre_tag_end         { CAT(block); DONE(block); fgoto main; };
    default => esc_pre;
  *|;

  pre_block := |*
    EOF { ADD_BLOCKCODE(); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCKCODE(); };
    double_return when not_extended { ADD_BLOCKCODE(); fgoto main; } ;
    double_return next_block_start when extended { ADD_EXTENDED_BLOCKCODE(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCKCODE(); fgoto main; };
    default => esc_pre;
  *|;

  script_tag := |*
    script_tag_end   { CAT(block); ASET("type", "ignore"); ADD_BLOCK(); fgoto main; };
    EOF              { ASET("type", "ignore"); ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  noparagraph_line := |*
    LF  { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  notextile_block := |*
    EOF { ADD_BLOCK(); fgoto main; };
    double_return when extended { CAT(block); ADD_EXTENDED_BLOCK(); CAT(html); };
    double_return when not_extended { ADD_BLOCK(); CAT(html); fgoto main; } ;
    double_return next_block_start when extended { CAT(block); ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); CAT(html); fgoto main; };
    default => cat;
  *|;

  html := |*
    html_end        { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  bc := |*
    EOF {  ADD_BLOCKCODE();  INLINE(html, "bc_close");  SET_PLAIN_BLOCK("p"); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCKCODE(); CAT(html); };
    double_return when not_extended { ADD_BLOCKCODE(); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCKCODE(); CAT(html); RSTRIP_BANG(html); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCKCODE(); INLINE(html, "bc_close"); SET_PLAIN_BLOCK("p"); fgoto main; };
    default => esc_pre;
  *|;

  bq := |*
    EOF { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCK(); };
    double_return when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCK(); INLINE(html, "bq_close"); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    html_end_terminating_block when extended { ADD_EXTENDED_BLOCK(); INLINE(html, "bq_close"); END_EXTENDED(); fgoto main; };
    html_end_terminating_block when not_extended { ADD_BLOCK(); INLINE(html, "bq_close"); fgoto main; };
    default => cat;
  *|;

  block := |*
    EOF { ADD_BLOCK(); fgoto main; };
    double_return when extended { ADD_EXTENDED_BLOCK(); };
    double_return when not_extended { ADD_BLOCK(); fgoto main; };
    double_return next_block_start when extended { ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    double_return next_block_start when not_extended { ADD_BLOCK(); fgoto main; };
    html_end_terminating_block when extended { ADD_EXTENDED_BLOCK(); END_EXTENDED(); fgoto main; };
    html_end_terminating_block when not_extended { ADD_BLOCK(); fgoto main; };
    LF list_start { ADD_BLOCK(); CLEAR_LIST(); LIST_LAYOUT(); fgoto list_item; };

    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  ul_item  = ( ul | ol )* ul A_HLGN* C :> " "+ ;
  ol_item  = ( ul | ol )* ol N_noactions A_HLGN* C :> " "+ ;
  list_item  := (" "* ( ul_item | ol_item )) @{ SET_ATTRIBUTES(); fgoto list_content; } ;

  list_content := |*
    LF list_start { ADD_BLOCK(); LIST_LAYOUT(); fgoto list_item; };
    block_end     { ADD_BLOCK(); RESET_NEST(); LIST_LAYOUT(); fgoto main; };
    default => cat;
  *|;

  dl := |*
    LF dt_start     { ADD_BLOCK(); ASET("type", "dt"); };
    dd_start        { ADD_BLOCK(); ASET("type", "dd"); };
    long_dd         { INLINE(html, "dd"); CLEAR_REGS(); };
    block_end       { ADD_BLOCK(); INLINE(html, "dl_close");  fgoto main; };
    default => cat;
  *|;

  main := |*
    noparagraph_line_start  { ASET("type", "ignored_line"); fgoto noparagraph_line; };
    notextile_tag   { INLINE(html, "notextile"); };
    notextile_block_start { ASET("type", "notextile"); fgoto notextile_block; };
    script_tag_start { CAT(block); fgoto script_tag; };
    pre_tag_start       { ASET("type", "notextile"); CAT(block); fgoto pre_tag; };
    pre_block_start { fgoto pre_block; };
    standalone_html { ASET("type", "html"); CAT(block); ADD_BLOCK(); };
    html_start      { ASET("type", "html_block"); fgoto html; };
    bc_start        { INLINE(html, "bc_open"); ASET("type", "code"); SET_PLAIN_BLOCK("code"); fgoto bc; };
    bq_start        { INLINE(html, "bq_open"); ASET("type", "p"); fgoto bq; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    horizontal_rule { INLINE(html, "hr"); };
    list_start      { CLEAR_LIST(); LIST_LAYOUT(); fgoto list_item; };
    dl_start        { fexec(ts + 1); INLINE(html, "dl_open"); ASET("type", "dt"); fgoto dl; };
    table           { INLINE(table, "table_close"); DONE(table); fgoto block; };
    link_alias      { STORE_LINK_ALIAS(); DONE(block); };
    aligned_image   { RESET_TYPE(); fgoto block; };
    redcloth_version { INLINE(html, "redcloth_version"); };
    blank_line => cat;
    default
    {
      CLEAR_REGS();
      RESET_TYPE();
      CAT(block);
      fgoto block;
    };
    EOF;
  *|;

}%%;


Changed text action from @{} to %{}.

Changed text action from @{} to %{}.

This ensures the action is only run at the end, opposed to any non final state.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback("on_comment", data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback("on_cdata", data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote* dquote);
    string_squote = (squote ^squote* squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple("on_element_start");
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple("on_element_end");
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = any* -- terminate_text;

    text := |*
        # Input such as just "</" or "<?". This rule takes precedence over the
        # rules below, but only if those don't match.
        terminate_text => {
            callback("on_text", data, encoding, ts, te);

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback("on_text", data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            fnext main;
        };

        # Just regular text.
        allowed_text => {
            callback("on_text", data, encoding, ts, te);
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback("on_comment", data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback("on_cdata", data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote* dquote);
    string_squote = (squote ^squote* squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple("on_element_start");
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple("on_element_end");
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = any* -- terminate_text;

    text := |*
        # Input such as just "</" or "<?". This rule takes precedence over the
        # rules below, but only if those don't match.
        terminate_text => {
            callback("on_text", data, encoding, ts, te);

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text @{ mark = p; } terminate_text => {
            callback("on_text", data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            fnext main;
        };

        # Just regular text.
        allowed_text => {
            callback("on_text", data, encoding, ts, te);
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%


add useful error messages for common parse conditions

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0);

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing");

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    ws*
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                    ws*
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec?
             $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); } ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0);

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing");

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}


partial check for parsing integer overflow

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

#define PARSE_ERROR(...) do { \
  snprintf(err_buf, err_buf_size, __VA_ARGS__); \
  err = 1; \
  goto bail; \
} while(0);

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if (curr_item.type == QSTRUCT_TYPE_BOOL && curr_item.fixed_array_size != 1)
        PARSE_ERROR("bools can't be arrays (line %d)", curr_line);

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld (line %d)", curr_item_index, curr_line);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld (line %d)", i, curr_line);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing (line %d)", curr_line);

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']'

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error (line %d)", curr_line);

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

#define PARSE_ERROR(...) do { \
  snprintf(err_buf, err_buf_size, __VA_ARGS__); \
  err = 1; \
  goto bail; \
} while(0);

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if (curr_item.type == QSTRUCT_TYPE_BOOL && curr_item.fixed_array_size != 1)
        PARSE_ERROR("bools can't be arrays (line %d)", curr_line);

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld (line %d)", curr_item_index, curr_line);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld (line %d)", i, curr_line);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing (line %d)", curr_line);

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']'

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error (line %d)", curr_line);

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}