Improve SSA parsing for files with stray [ in them.

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize {[delegate releaseSpanExtra:ex]; [super finalize];}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl :> (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl :> (format %{eventformat=str;})? <: (event*);

		main := bom? header styles lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = a, *divB = b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
					div->positioned = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" "-"? [0-9]+ "," "-"? [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize {[delegate releaseSpanExtra:ex]; [super finalize];}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl (format %{eventformat=str;})? <: (event*);

		main := bom? header :> styles :> lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = a, *divB = b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
					div->positioned = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" "-"? [0-9]+ "," "-"? [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}


Fix(Lexer) use Ruby #empty? instead of Rails #blank?

%%{
  machine graphql_lexer;

  IDENTIFIER =    [_A-Za-z][_0-9A-Za-z]*;
  NEWLINE =       [\c\r\n];
  BLANK   =       [, \t]+;
  COMMENT =       '#' [^\n\r]*;
  INT =           '-'? ('0'|[1-9][0-9]*);
  FLOAT_DECIMAL = '.'[0-9]+;
  FLOAT_EXP =     ('e' | 'E')?('+' | '-')?[0-9]+;
  FLOAT =         INT FLOAT_DECIMAL? FLOAT_EXP?;
  ON =            'on';
  FRAGMENT =      'fragment';
  TRUE =          'true';
  FALSE =         'false';
  NULL =          'null';
  QUERY =         'query';
  MUTATION =      'mutation';
  SUBSCRIPTION =  'subscription';
  SCHEMA =        'schema';
  SCALAR =        'scalar';
  TYPE =          'type';
  IMPLEMENTS =    'implements';
  INTERFACE =     'interface';
  UNION =         'union';
  ENUM =          'enum';
  INPUT =         'input';
  DIRECTIVE =     'directive';
  LCURLY =        '{';
  RCURLY =        '}';
  LPAREN =        '(';
  RPAREN =        ')';
  LBRACKET =      '[';
  RBRACKET =      ']';
  COLON =         ':';
  QUOTE =         '"';
  BLOCK_QUOTE =   '"""';
  ESCAPED_BLOCK_QUOTE = '\\"""';
  BLOCK_STRING_CHAR = (ESCAPED_BLOCK_QUOTE | ^'"""');
  ESCAPED_QUOTE = '\\"';
  STRING_CHAR =   (ESCAPED_QUOTE | ^'"');
  VAR_SIGN =      '$';
  DIR_SIGN =      '@';
  ELLIPSIS =      '...';
  EQUALS =        '=';
  BANG =          '!';
  PIPE =          '|';

  QUOTED_STRING = QUOTE STRING_CHAR* QUOTE;
  BLOCK_STRING = BLOCK_QUOTE BLOCK_STRING_CHAR* BLOCK_QUOTE;
  # catch-all for anything else. must be at the bottom for precedence.
  UNKNOWN_CHAR =         /./;

  main := |*
    INT           => { emit(:INT, ts, te, meta) };
    FLOAT         => { emit(:FLOAT, ts, te, meta) };
    ON            => { emit(:ON, ts, te, meta) };
    FRAGMENT      => { emit(:FRAGMENT, ts, te, meta) };
    TRUE          => { emit(:TRUE, ts, te, meta) };
    FALSE         => { emit(:FALSE, ts, te, meta) };
    NULL          => { emit(:NULL, ts, te, meta) };
    QUERY         => { emit(:QUERY, ts, te, meta) };
    MUTATION      => { emit(:MUTATION, ts, te, meta) };
    SUBSCRIPTION  => { emit(:SUBSCRIPTION, ts, te, meta) };
    SCHEMA        => { emit(:SCHEMA, ts, te, meta) };
    SCALAR        => { emit(:SCALAR, ts, te, meta) };
    TYPE          => { emit(:TYPE, ts, te, meta) };
    IMPLEMENTS    => { emit(:IMPLEMENTS, ts, te, meta) };
    INTERFACE     => { emit(:INTERFACE, ts, te, meta) };
    UNION         => { emit(:UNION, ts, te, meta) };
    ENUM          => { emit(:ENUM, ts, te, meta) };
    INPUT         => { emit(:INPUT, ts, te, meta) };
    DIRECTIVE     => { emit(:DIRECTIVE, ts, te, meta) };
    RCURLY        => { emit(:RCURLY, ts, te, meta) };
    LCURLY        => { emit(:LCURLY, ts, te, meta) };
    RPAREN        => { emit(:RPAREN, ts, te, meta) };
    LPAREN        => { emit(:LPAREN, ts, te, meta) };
    RBRACKET      => { emit(:RBRACKET, ts, te, meta) };
    LBRACKET      => { emit(:LBRACKET, ts, te, meta) };
    COLON         => { emit(:COLON, ts, te, meta) };
    QUOTED_STRING => { emit_string(ts, te, meta, block: false) };
    BLOCK_STRING  => { emit_string(ts, te, meta, block: true) };
    VAR_SIGN      => { emit(:VAR_SIGN, ts, te, meta) };
    DIR_SIGN      => { emit(:DIR_SIGN, ts, te, meta) };
    ELLIPSIS      => { emit(:ELLIPSIS, ts, te, meta) };
    EQUALS        => { emit(:EQUALS, ts, te, meta) };
    BANG          => { emit(:BANG, ts, te, meta) };
    PIPE          => { emit(:PIPE, ts, te, meta) };
    IDENTIFIER    => { emit(:IDENTIFIER, ts, te, meta) };
    COMMENT       => { record_comment(ts, te, meta) };

    NEWLINE => {
      meta[:line] += 1
      meta[:col] = 1
    };

    BLANK   => { meta[:col] += te - ts };

    UNKNOWN_CHAR => { emit(:UNKNOWN_CHAR, ts, te, meta) };

  *|;
}%%

module GraphQL
  module Language
    module Lexer
      def self.tokenize(query_string)
        run_lexer(query_string)
      end

      # Replace any escaped unicode or whitespace with the _actual_ characters
      # To avoid allocating more strings, this modifies the string passed into it
      def self.replace_escaped_characters_in_place(raw_string)
        raw_string.gsub!(ESCAPES, ESCAPES_REPLACE)
        raw_string.gsub!(UTF_8, &UTF_8_REPLACE)
        nil
      end

      private

      %% write data;

      def self.run_lexer(query_string)
        data = query_string.unpack("c*")
        eof = data.length

        # Since `Lexer` is a module, store all lexer state
        # in this local variable:
        meta = {
          line: 1,
          col: 1,
          data: data,
          tokens: [],
          previous_token: nil,
        }

        p ||= 0
        pe ||= data.length

        %% write init;

        %% write exec;

        meta[:tokens]
      end

      def self.record_comment(ts, te, meta)
        token = GraphQL::Language::Token.new(
          name: :COMMENT,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )

        meta[:previous_token] = token

        meta[:col] += te - ts
      end

      def self.emit(token_name, ts, te, meta)
        meta[:tokens] << token = GraphQL::Language::Token.new(
          name: token_name,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )
        meta[:previous_token] = token
        # Bump the column counter for the next token
        meta[:col] += te - ts
      end

      ESCAPES = /\\["\\\/bfnrt]/
      ESCAPES_REPLACE = {
        '\\"' => '"',
        "\\\\" => "\\",
        "\\/" => '/',
        "\\b" => "\b",
        "\\f" => "\f",
        "\\n" => "\n",
        "\\r" => "\r",
        "\\t" => "\t",
      }

      UTF_8 = /\\u[\dAa-f]{4}/i
      UTF_8_REPLACE = ->(m) { [m[-4..-1].to_i(16)].pack('U'.freeze) }

      VALID_STRING = /\A(?:[^\\]|#{ESCAPES}|#{UTF_8})*\z/o

      PACK_DIRECTIVE = "c*"
      UTF_8_ENCODING = "UTF-8"

      def self.emit_string(ts, te, meta, block:)
        quotes_length = block ? 3 : 1
        ts += quotes_length
        value = meta[:data][ts...te - quotes_length].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING)
        if block
          value = trim_indent(value)
        end
        if value !~ VALID_STRING
          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :BAD_UNICODE_ESCAPE,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        else
          replace_escaped_characters_in_place(value)

          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :STRING,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        end

        meta[:previous_token] = token
        meta[:col] += te - ts
      end

      def self.trim_indent(str)
        lines = str.split("\n")
        common_indent = nil

        # find the common whitespace
        lines.each_with_index do |line, idx|
          if idx == 0
            next
          end
          line_length = line.size
          line_indent = line[/\A */].size
          if line_indent < line_length && common_indent.nil? || line_indent < common_indent
            common_indent = line_indent
          end
        end

        # Remove the common whitespace
        if common_indent
          lines.each_with_index do |line, idx|
            if idx == 0
              next
            else
              line[0, common_indent] = ""
            end
          end
        end

        # Remove leading & trailing blank lines
        while lines.first.empty?
          lines.shift
        end
        while lines.last.empty?
          lines.pop
        end

        # Rebuild the string
        lines.join("\n")
      end
    end
  end
end

%%{
  machine graphql_lexer;

  IDENTIFIER =    [_A-Za-z][_0-9A-Za-z]*;
  NEWLINE =       [\c\r\n];
  BLANK   =       [, \t]+;
  COMMENT =       '#' [^\n\r]*;
  INT =           '-'? ('0'|[1-9][0-9]*);
  FLOAT_DECIMAL = '.'[0-9]+;
  FLOAT_EXP =     ('e' | 'E')?('+' | '-')?[0-9]+;
  FLOAT =         INT FLOAT_DECIMAL? FLOAT_EXP?;
  ON =            'on';
  FRAGMENT =      'fragment';
  TRUE =          'true';
  FALSE =         'false';
  NULL =          'null';
  QUERY =         'query';
  MUTATION =      'mutation';
  SUBSCRIPTION =  'subscription';
  SCHEMA =        'schema';
  SCALAR =        'scalar';
  TYPE =          'type';
  IMPLEMENTS =    'implements';
  INTERFACE =     'interface';
  UNION =         'union';
  ENUM =          'enum';
  INPUT =         'input';
  DIRECTIVE =     'directive';
  LCURLY =        '{';
  RCURLY =        '}';
  LPAREN =        '(';
  RPAREN =        ')';
  LBRACKET =      '[';
  RBRACKET =      ']';
  COLON =         ':';
  QUOTE =         '"';
  BLOCK_QUOTE =   '"""';
  ESCAPED_BLOCK_QUOTE = '\\"""';
  BLOCK_STRING_CHAR = (ESCAPED_BLOCK_QUOTE | ^'"""');
  ESCAPED_QUOTE = '\\"';
  STRING_CHAR =   (ESCAPED_QUOTE | ^'"');
  VAR_SIGN =      '$';
  DIR_SIGN =      '@';
  ELLIPSIS =      '...';
  EQUALS =        '=';
  BANG =          '!';
  PIPE =          '|';

  QUOTED_STRING = QUOTE STRING_CHAR* QUOTE;
  BLOCK_STRING = BLOCK_QUOTE BLOCK_STRING_CHAR* BLOCK_QUOTE;
  # catch-all for anything else. must be at the bottom for precedence.
  UNKNOWN_CHAR =         /./;

  main := |*
    INT           => { emit(:INT, ts, te, meta) };
    FLOAT         => { emit(:FLOAT, ts, te, meta) };
    ON            => { emit(:ON, ts, te, meta) };
    FRAGMENT      => { emit(:FRAGMENT, ts, te, meta) };
    TRUE          => { emit(:TRUE, ts, te, meta) };
    FALSE         => { emit(:FALSE, ts, te, meta) };
    NULL          => { emit(:NULL, ts, te, meta) };
    QUERY         => { emit(:QUERY, ts, te, meta) };
    MUTATION      => { emit(:MUTATION, ts, te, meta) };
    SUBSCRIPTION  => { emit(:SUBSCRIPTION, ts, te, meta) };
    SCHEMA        => { emit(:SCHEMA, ts, te, meta) };
    SCALAR        => { emit(:SCALAR, ts, te, meta) };
    TYPE          => { emit(:TYPE, ts, te, meta) };
    IMPLEMENTS    => { emit(:IMPLEMENTS, ts, te, meta) };
    INTERFACE     => { emit(:INTERFACE, ts, te, meta) };
    UNION         => { emit(:UNION, ts, te, meta) };
    ENUM          => { emit(:ENUM, ts, te, meta) };
    INPUT         => { emit(:INPUT, ts, te, meta) };
    DIRECTIVE     => { emit(:DIRECTIVE, ts, te, meta) };
    RCURLY        => { emit(:RCURLY, ts, te, meta) };
    LCURLY        => { emit(:LCURLY, ts, te, meta) };
    RPAREN        => { emit(:RPAREN, ts, te, meta) };
    LPAREN        => { emit(:LPAREN, ts, te, meta) };
    RBRACKET      => { emit(:RBRACKET, ts, te, meta) };
    LBRACKET      => { emit(:LBRACKET, ts, te, meta) };
    COLON         => { emit(:COLON, ts, te, meta) };
    QUOTED_STRING => { emit_string(ts, te, meta, block: false) };
    BLOCK_STRING  => { emit_string(ts, te, meta, block: true) };
    VAR_SIGN      => { emit(:VAR_SIGN, ts, te, meta) };
    DIR_SIGN      => { emit(:DIR_SIGN, ts, te, meta) };
    ELLIPSIS      => { emit(:ELLIPSIS, ts, te, meta) };
    EQUALS        => { emit(:EQUALS, ts, te, meta) };
    BANG          => { emit(:BANG, ts, te, meta) };
    PIPE          => { emit(:PIPE, ts, te, meta) };
    IDENTIFIER    => { emit(:IDENTIFIER, ts, te, meta) };
    COMMENT       => { record_comment(ts, te, meta) };

    NEWLINE => {
      meta[:line] += 1
      meta[:col] = 1
    };

    BLANK   => { meta[:col] += te - ts };

    UNKNOWN_CHAR => { emit(:UNKNOWN_CHAR, ts, te, meta) };

  *|;
}%%

module GraphQL
  module Language
    module Lexer
      def self.tokenize(query_string)
        run_lexer(query_string)
      end

      # Replace any escaped unicode or whitespace with the _actual_ characters
      # To avoid allocating more strings, this modifies the string passed into it
      def self.replace_escaped_characters_in_place(raw_string)
        raw_string.gsub!(ESCAPES, ESCAPES_REPLACE)
        raw_string.gsub!(UTF_8, &UTF_8_REPLACE)
        nil
      end

      private

      %% write data;

      def self.run_lexer(query_string)
        data = query_string.unpack("c*")
        eof = data.length

        # Since `Lexer` is a module, store all lexer state
        # in this local variable:
        meta = {
          line: 1,
          col: 1,
          data: data,
          tokens: [],
          previous_token: nil,
        }

        p ||= 0
        pe ||= data.length

        %% write init;

        %% write exec;

        meta[:tokens]
      end

      def self.record_comment(ts, te, meta)
        token = GraphQL::Language::Token.new(
          name: :COMMENT,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )

        meta[:previous_token] = token

        meta[:col] += te - ts
      end

      def self.emit(token_name, ts, te, meta)
        meta[:tokens] << token = GraphQL::Language::Token.new(
          name: token_name,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )
        meta[:previous_token] = token
        # Bump the column counter for the next token
        meta[:col] += te - ts
      end

      ESCAPES = /\\["\\\/bfnrt]/
      ESCAPES_REPLACE = {
        '\\"' => '"',
        "\\\\" => "\\",
        "\\/" => '/',
        "\\b" => "\b",
        "\\f" => "\f",
        "\\n" => "\n",
        "\\r" => "\r",
        "\\t" => "\t",
      }

      UTF_8 = /\\u[\dAa-f]{4}/i
      UTF_8_REPLACE = ->(m) { [m[-4..-1].to_i(16)].pack('U'.freeze) }

      VALID_STRING = /\A(?:[^\\]|#{ESCAPES}|#{UTF_8})*\z/o

      PACK_DIRECTIVE = "c*"
      UTF_8_ENCODING = "UTF-8"

      def self.emit_string(ts, te, meta, block:)
        quotes_length = block ? 3 : 1
        ts += quotes_length
        value = meta[:data][ts...te - quotes_length].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING)
        if block
          value = trim_indent(value)
        end
        if value !~ VALID_STRING
          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :BAD_UNICODE_ESCAPE,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        else
          replace_escaped_characters_in_place(value)

          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :STRING,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        end

        meta[:previous_token] = token
        meta[:col] += te - ts
      end

      def self.trim_indent(str)
        lines = str.split("\n")
        common_indent = nil

        # find the common whitespace
        lines.each_with_index do |line, idx|
          if idx == 0
            next
          end
          line_length = line.size
          line_indent = line[/\A */].size
          if line_indent < line_length && common_indent.nil? || line_indent < common_indent
            common_indent = line_indent
          end
        end

        # Remove the common whitespace
        if common_indent
          lines.each_with_index do |line, idx|
            if idx == 0
              next
            else
              line[0, common_indent] = ""
            end
          end
        end

        # Remove leading & trailing blank lines
        while lines.first.blank?
          lines.shift
        end
        while lines.last.blank?
          lines.pop
        end

        # Rebuild the string
        lines.join("\n")
      end
    end
  end
end


hext-machine: fix formatting

hext-machine: fix formatting

%%{

# This ragel machine parses/lexes hext. It is embedded in ragel/Parser.cpp.rl.
machine hext;

#### NTH_PATTERN ###############################################################
# nth_pattern is used by traits, e.g. nth-last-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  ( 'even' %{ pv.nth = {2, 0}; } )
  |

  ( 'odd' %{ pv.nth = {2, 1}; } )
  |

  ( ( ( '-'? [0-9]+ )
      >{ TK_START; }
      %{ TK_STOP; pv.nth = {0, std::stoi(tok)}; }
    )
    ( 'n'
      %{ pv.nth = {pv.nth.second, 0}; }
      ( ( ('+'|'-') [0-9]+ )
        >{ TK_START; }
        %{ TK_STOP; pv.nth.second = std::stoi(tok); }
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  ( 'empty' %{ pv.set_trait<ChildCountMatch>(0); } )
  |

  ( 'child-count('
    ( [0-9]+
      >{ TK_START; }
      %{ TK_STOP; pv.set_trait<ChildCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'attribute-count('
    ( [0-9]+
      >{ TK_START; }
      %{ TK_STOP; pv.set_trait<AttributeCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'nth-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth); } )
  |

  ( 'nth-last-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last); } )
  |

  ( 'nth-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'first-child'
    %{ pv.set_trait<NthChildMatch>(0, 1); } )
  |

  ( 'first-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'last-child'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last); } )
  |

  ( 'last-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'only-child' %{ pv.set_trait<OnlyChildMatch>(); } )
  |

  ( 'only-of-type' %{ pv.set_trait<OnlyChildMatch>(OnlyChildMatch::OfType); } )
  |

  ( 'text' %{ pv.set_trait<TextNodeMatch>(); } )
);
not_trait =
':not('
  %{ pv.negate = std::make_unique<NegateMatch>(); }
  (
    trait
    %{
       assert(pv.negate);
       pv.negate->take_match(std::move(pv.trait));
     }
  )+
')';

#### REGULAR EXPRESSIONS #######################################################
regex =
'/' ( ( [^/] | '\\/' )** >{ TK_START; } %{ TK_STOP; } ) '/'
# regex flags
(
  # case insensitive
  ( 'i' %{ pv.regex_flag |= boost::regex::icase; } )
  |
  # collate (locale aware character groups)
  ( 'c' %{ pv.regex_flag |= boost::regex::collate; } )
)*
%{ try {
     pv.regex = boost::regex(tok, pv.regex_flag);
   }
   catch( const boost::regex_error& e ) {
     // Mark whole regex as error, including slashes and flags
     auto mark_len = static_cast<std::size_t>(this->p - tok_begin + 1);
     this->throw_regex_error(mark_len, e.code());
   }
};
# Wrap a regular expression into a RegexTest.
regex_test = regex %{ assert(pv.regex); pv.set_test<RegexTest>(*pv.regex); };

#### BUILTIN FUNCTION ##########################################################
builtin = (
  '@'
  ( ( 'text'       %{ pv.builtin = TextBuiltin; } )
    |
    ( 'inner-html' %{ pv.builtin = InnerHtmlBuiltin; } )
    |
    ( 'strip-tags' %{ pv.builtin = StripTagsBuiltin; } ) )
);

#### CAPTURE ###################################################################
# capture variable, e.g. id={cap_var}, @text={cap_var}
capture = (
  '{' ( ( [^/{}][^/{}]** )
        >{ TK_START; }
        %{ TK_STOP; pv.cap_var = tok; } )
      # optional capture regex, e.g. @text={time/(\d\d:\d\d)/}
      regex?
  '}'
);

#### HELPERS ###################################################################
# Set an attribute capture to optional. By default, attribute captures are
# mandatory, i.e. the attribute has to exist in a node for it to match.
optional = (
  '?' %{ pv.optional = true; }
);
# Negate a match pattern, e.g. style!, class="menu"!, @text=/foo/!
negate = (
  '!' %{ pv.set_test<NegateTest>(std::move(pv.test)); }
);
# The name of an HTML-element's attribute
attr_name = (
  ( alpha (alnum | '-' | '_')** )
  >{ TK_START; }
  %{ TK_STOP; pv.attr_name = tok; }
);
# Literal value, e.g. attr="literal", class='single'
literal_value =
( '"' (
    ( [^"] | '\\"' )**
    >{ TK_START; } %{ TK_STOP; pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( [^'] | '\\\'' )**
    >{ TK_START; } %{ TK_STOP; pv.literal_value = tok; }
  ) '\'' );

#### LITERAL ###################################################################
literal = (
  ( '=' literal_value
    %{ pv.set_test<ContainsWordsTest>(pv.literal_value); } )
  |
  ( '==' literal_value
    %{ pv.set_test<EqualsTest>(pv.literal_value); } )
  |
  ( '^=' literal_value
    %{ pv.set_test<BeginsWithTest>(pv.literal_value); } )
  |
  ( '*=' literal_value
    %{ pv.set_test<ContainsTest>(pv.literal_value); } )
  |
  ( '$=' literal_value
    %{ pv.set_test<EndsWithTest>(pv.literal_value); } )
);

#### PATTERNS ##################################################################
pattern = (
  space+
  ( ( ( builtin '=' capture )
      %{ if( pv.regex )
           cur_rule().append_capture<FunctionCapture>(pv.builtin, pv.cap_var, *pv.regex);
         else
           cur_rule().append_capture<FunctionCapture>(pv.builtin, pv.cap_var); } )
    |

    ( ( builtin '=' regex_test negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( builtin literal negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( attr_name '=' capture optional? )
      %{ if( pv.regex )
           cur_rule()
             .append_capture<AttributeCapture>(pv.attr_name, pv.cap_var, *pv.regex);
         else
           cur_rule().append_capture<AttributeCapture>(pv.attr_name, pv.cap_var);
         if( !pv.optional )
           cur_rule().append_match<AttributeMatch>(pv.attr_name);
       } )
    |

    ( ( attr_name '=' regex_test negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name literal negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name
        %{ pv.test = nullptr; }
        negate?
      )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
  ) %{ pv.reset(); }
);

#### RULES #####################################################################
tag_name = ( alpha (alnum | '-' | '_')** );
main := (
  # ignore whitespace
  space
  |

  # comment
  ( '#' (any - '\n')* '\n' )
  |

  # open rule
  (
    # a rule starts with '<'
    ( '<' %{ push_rule(); } )

    # a rule can be optional
    ( '?' %{ cur_rule().set_optional(true); } )?

    # a rule must have a tag name, e.g. <div
    ( ( '*' | tag_name ) >{ TK_START; }
                         %{ TK_STOP; set_open_tag_or_throw(tok); } )

    # a rule can have multiple traits, e.g. :first-child, :empty
    ( ( not_trait %{ cur_rule().append_match(std::move(pv.negate)); } )
      |

      ( trait %{ cur_rule().append_match(std::move(pv.trait)); } ) )*

    # a rule can have multiple match or capture patterns,
    # e.g. class="menu", @text={heading}
    pattern*

    space*

    ( ( '/>' %{ pop_rule(); } ) | '>' )
  )
  |

  # end rule
  (
    '</'
    ( ( '*' | tag_name ) >{ TK_START; }
                         %{ TK_STOP;
                            validate_close_tag_or_throw(tok);
                            pop_rule(); } )
    '>'
  )
)* $err{ this->throw_unexpected(); };

}%%

%%{

# This ragel machine parses/lexes hext. It is embedded in ragel/Parser.cpp.rl.
machine hext;

#### NTH_PATTERN ###############################################################
# nth_pattern is used by traits, e.g. nth-last-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  ( 'even' %{ pv.nth = {2, 0}; } )
  |

  ( 'odd' %{ pv.nth = {2, 1}; } )
  |

  ( ( ( '-'? [0-9]+ )
      >{ TK_START; }
      %{ TK_STOP; pv.nth = {0, std::stoi(tok)}; }
    )
    ( 'n'
      %{ pv.nth = {pv.nth.second, 0}; }
      ( ( ('+'|'-') [0-9]+ )
        >{ TK_START; }
        %{ TK_STOP; pv.nth.second = std::stoi(tok); }
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  ( 'empty' %{ pv.set_trait<ChildCountMatch>(0); } )
  |

  ( 'child-count('
    ( [0-9]+
      >{ TK_START; }
      %{ TK_STOP; pv.set_trait<ChildCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'attribute-count('
    ( [0-9]+
      >{ TK_START; }
      %{ TK_STOP; pv.set_trait<AttributeCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'nth-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth); } )
  |

  ( 'nth-last-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last); } )
  |

  ( 'nth-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'first-child'
    %{ pv.set_trait<NthChildMatch>(0, 1); } )
  |

  ( 'first-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'last-child'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last); } )
  |

  ( 'last-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'only-child' %{ pv.set_trait<OnlyChildMatch>(); } )
  |

  ( 'only-of-type' %{ pv.set_trait<OnlyChildMatch>(OnlyChildMatch::OfType); } )
  |

  ( 'text' %{ pv.set_trait<TextNodeMatch>(); } )
);
not_trait =
':not('
  %{ pv.negate = std::make_unique<NegateMatch>(); }
  (
    trait
    %{
       assert(pv.negate);
       pv.negate->take_match(std::move(pv.trait));
     }
  )+
')';

#### REGULAR EXPRESSIONS #######################################################
regex =
'/' ( ( [^/] | '\\/' )** >{ TK_START; } %{ TK_STOP; } ) '/'
# regex flags
(
  # case insensitive
  ( 'i' %{ pv.regex_flag |= boost::regex::icase; } )
  |
  # collate (locale aware character groups)
  ( 'c' %{ pv.regex_flag |= boost::regex::collate; } )
)*
%{ try {
     pv.regex = boost::regex(tok, pv.regex_flag);
   }
   catch( const boost::regex_error& e ) {
     // Mark whole regex as error, including slashes and flags
     auto mark_len = static_cast<std::size_t>(this->p - tok_begin + 1);
     this->throw_regex_error(mark_len, e.code());
   }
};
# Wrap a regular expression into a RegexTest.
regex_test = regex %{ assert(pv.regex); pv.set_test<RegexTest>(*pv.regex); };

#### BUILTIN FUNCTION ##########################################################
builtin = (
  '@'
  ( ( 'text'       %{ pv.builtin = TextBuiltin; } )
    |
    ( 'inner-html' %{ pv.builtin = InnerHtmlBuiltin; } )
    |
    ( 'strip-tags' %{ pv.builtin = StripTagsBuiltin; } ) )
);

#### CAPTURE ###################################################################
# capture variable, e.g. id={cap_var}, @text={cap_var}
capture = (
  '{' ( ( [^/{}][^/{}]** )
        >{ TK_START; }
        %{ TK_STOP; pv.cap_var = tok; } )
      # optional capture regex, e.g. @text={time/(\d\d:\d\d)/}
      regex?
  '}'
);

#### HELPERS ###################################################################
# Set an attribute capture to optional. By default, attribute captures are
# mandatory, i.e. the attribute has to exist in a node for it to match.
optional = (
  '?' %{ pv.optional = true; }
);
# Negate a match pattern, e.g. style!, class="menu"!, @text=/foo/!
negate = (
  '!' %{ pv.set_test<NegateTest>(std::move(pv.test)); }
);
# The name of an HTML-element's attribute
attr_name = (
  ( alpha (alnum | '-' | '_')** )
  >{ TK_START; }
  %{ TK_STOP; pv.attr_name = tok; }
);
# Literal value, e.g. attr="literal", class='single'
literal_value =
( '"' (
    ( [^"] | '\\"' )**
    >{ TK_START; } %{ TK_STOP; pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( [^'] | '\\\'' )**
    >{ TK_START; } %{ TK_STOP; pv.literal_value = tok; }
  ) '\'' );

#### LITERAL ###################################################################
literal = (
  ( '=' literal_value
    %{ pv.set_test<ContainsWordsTest>(pv.literal_value); } )
  |
  ( '==' literal_value
    %{ pv.set_test<EqualsTest>(pv.literal_value); } )
  |
  ( '^=' literal_value
    %{ pv.set_test<BeginsWithTest>(pv.literal_value); } )
  |
  ( '*=' literal_value
    %{ pv.set_test<ContainsTest>(pv.literal_value); } )
  |
  ( '$=' literal_value
    %{ pv.set_test<EndsWithTest>(pv.literal_value); } )
);

#### PATTERNS ##################################################################
pattern = (
  space+
  ( ( ( builtin '=' capture )
      %{ if( pv.regex )
           cur_rule().append_capture<FunctionCapture>(pv.builtin, pv.cap_var, *pv.regex);
         else
           cur_rule().append_capture<FunctionCapture>(pv.builtin, pv.cap_var); } )
    |

    ( ( builtin '=' regex_test negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( builtin literal negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( attr_name '=' capture optional? )
      %{ if( pv.regex )
           cur_rule()
             .append_capture<AttributeCapture>(pv.attr_name, pv.cap_var, *pv.regex);
         else
           cur_rule().append_capture<AttributeCapture>(pv.attr_name, pv.cap_var);
         if( !pv.optional )
           cur_rule().append_match<AttributeMatch>(pv.attr_name);
       } )
    |

    ( ( attr_name '=' regex_test negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name literal negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name
        %{ pv.test = nullptr; }
        negate?
      )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
  ) %{ pv.reset(); }
);

#### RULES #####################################################################
tag_name = ( alpha (alnum | '-' | '_')** );
main := (
  # ignore whitespace
  space
  |

  # comment
  ( '#' (any - '\n')* '\n' )
  |

  # open rule
  (
    # a rule starts with '<'
    ( '<' %{ push_rule(); } )

    # a rule can be optional, i.e. starts with <?
    ( '?' %{ cur_rule().set_optional(true); } )?

    # a rule must have a tag name, e.g. <div
    ( ('*' | tag_name ) >{ TK_START; }
                        %{ TK_STOP; set_open_tag_or_throw(tok); } )

    # a rule can have multiple traits, e.g. :first-child, :empty
    ( ( not_trait %{ cur_rule().append_match(std::move(pv.negate)); } )
      |

      ( trait %{ cur_rule().append_match(std::move(pv.trait)); } ) )*

    # a rule can have multiple match or capture patterns,
    # e.g. class="menu", @text={heading}
    pattern*

    space*

    ( ( '/>' %{ pop_rule(); } ) | '>' )
  )
  |

  # end rule
  (
    '</'
    ( ( '*' | tag_name ) >{ TK_START; }
                         %{ TK_STOP;
                            validate_close_tag_or_throw(tok);
                            pop_rule(); } )
    '>'
  )
)* $err{ this->throw_unexpected(); };

}%%


make control "kill" command use Register_disconnect() rather than fdclose().

Previously it would leave a stale registration entry, and any new connections
that re-used that fd would get booted immediately.

#include "control.h"
#include "bstring.h"
#include "task/task.h"
#include "register.h"
#include "dbg.h"
#include <stdlib.h>
#include <time.h>
#include "setting.h"
#include <signal.h>

void bstring_free(void *data, void *hint)
{
    bdestroy((bstring)hint);
}

struct tagbstring REQ_STATUS_TASKS = bsStatic("status tasks");
struct tagbstring REQ_STATUS_NET = bsStatic("status net");

static inline bstring read_message(void *sock)
{
    bstring req = NULL;
    int rc = 0;

    zmq_msg_t *inmsg = calloc(sizeof(zmq_msg_t), 1);
    rc = zmq_msg_init(inmsg);
    check(rc == 0, "init failed.");

    rc = mqrecv(sock, inmsg, ZMQ_NOBLOCK);
    check(rc == 0, "Receive on control failed.");

    req = blk2bstr(zmq_msg_data(inmsg), zmq_msg_size(inmsg));
    check(req, "Failed to create the request string.");

    zmq_msg_close(inmsg);
    free(inmsg);

    return req;

error:
    if(inmsg) {
        zmq_msg_close(inmsg);
        free(inmsg);
    }

    return NULL;
}

static inline void send_reply(void *sock, bstring rep)
{
    int rc = 0;
    zmq_msg_t *outmsg = calloc(sizeof(zmq_msg_t), 1);
    rc = zmq_msg_init(outmsg);
    check(rc == 0, "init failed.");

    rc = zmq_msg_init_data(outmsg, bdata(rep), blength(rep), bstring_free, rep);
    check(rc == 0, "Failed to init reply data.");

    rc = mqsend(sock, outmsg, 0);
    check(rc == 0, "Failed to deliver 0mq message to requestor.");

error:
    free(outmsg);
    return;
}

static int CONTROL_RUNNING = 1;
static void *CONTROL_SOCKET = NULL;

%%{
    machine ControlParser;

    action mark { mark = fpc; }

    action status_tasks { reply = taskgetinfo(); fbreak; }
    action status_net { reply = Register_info(); fbreak; }

    action control_stop {
        reply = bfromcstr("{\"msg\": \"stopping control port\"}");
        CONTROL_RUNNING = 0; fbreak;
    }

    action time {
        reply = bformat("{\"time\": %d}", (int)time(NULL)); fbreak;
    }

    action kill {
        int id = atoi(p);

        if(id < 0 || id > MAX_REGISTERED_FDS) {
            reply = bformat("{\"error\": \"invalid id: %d\"}", id);
        } else {
            int fd = Register_fd_for_id(id);

            if(fd >= 0) {
                Register_disconnect(fd);
                reply = bformat("{\"result\": \"killed %d\"}", id);
            } else {
                reply = bformat("{\"error\": \"does not exist: %d\"}", id);
            }
        }

        fbreak;
    }

    action reload {
        int rc = raise(SIGHUP);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be reloaded\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to reload the server\"}");
        }
        fbreak;
    }

    action stop {
        // TODO: probably report back the number of waiting tasks
        int rc = raise(SIGINT);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be stopped\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to stop the server\"}");
        }
        fbreak;
    }

    action terminate {
        // TODO: the server might have been terminated before
        // the reply is sent back. If this scenario is crucial (if possible at all)
        // find a workaround, otherwise, forget about it and remove this comment.
        int rc = raise(SIGTERM);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be terminated\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to terminate the server\"}");
        }
        fbreak;
    }

    action help {
        reply = bfromcstr("{\"command_list\":[");
        bcatcstr(reply, "{\"name\": \"control stop\", \"description\": \"Close the control port.\"},\n");
        bcatcstr(reply, "{\"name\": \"kill <id>\", \"description\": \"Kill a connection in a violent way.\"},\n");
        bcatcstr(reply, "{\"name\": \"reload\", \"description\": \"Reload the server. Same as `kill -SIGHUP <server_pid>`.\"},\n");
        bcatcstr(reply, "{\"name\": \"status net\", \"description\": \"Return a list of active connections.\"},\n");
        bcatcstr(reply, "{\"name\": \"status tasks\", \"description\": \"Return a list of active tasks.\"},\n");
        bcatcstr(reply, "{\"name\": \"stop\", \"description\": \"Gracefully shut down the server. Same as `kill -SIGINT <server_pid>`.\"},\n");
        bcatcstr(reply, "{\"name\": \"terminate\", \"description\": \"Unconditionally terminate the server. Same as `kill -SIGTERM <server_pid>.\"},\n");
        bcatcstr(reply, "{\"name\": \"time\", \"description\": \"Return server timestamp.\"}\n");
        bcatcstr(reply, "]}\n");
        fbreak;
    }

    Status = "status" space+ ("tasks" @status_tasks | "net" @status_net);
    Control = "control stop" @control_stop;
    Time = "time" @time;
    Kill = "kill" space+ digit+ >mark @kill;
    Reload = "reload" @reload;
    Stop = "stop" @stop;
    Terminate = "terminate" @terminate;
    Help = "help" @help;

    main := Status | Control | Time | Kill | Reload | Stop | Terminate | Help;
}%%

%% write data;

bstring Control_execute(bstring req)
{
    const char *p = bdata(req);
    const char *pe = p+blength(req);
    const char *mark = NULL;
    int cs = 0;
    bstring reply = NULL;

    debug("RECEIVED CONTROL COMMAND: %s", bdata(req));

    %% write init;
    %% write exec noend;

    check(p <= pe, "Buffer overflow after parsing.  Tell Zed that you sent something from a handler that went %ld past the end in the parser.",
        (long int)(pe - p));

    if ( cs == %%{ write error; }%% ) {
        check(pe - p > 0, "Major erorr in the parser, tell Zed.");
        return bformat("{\"error\": \"parsing error at: ...%s\"}", bdata(req) + (pe - p));
    } else if(!reply) {
        return bformat("{\"error\": \"invalid command\"}", pe - p);
    } else {
        return reply;
    }

error:
    return bfromcstr("{\"error\": \"fatal error\"}");
}

struct tagbstring DEFAULT_CONTROL_SPEC = bsStatic("ipc://run/control");

void Control_task(void *v)
{
    int rc = 0;
    bstring req = NULL;
    bstring rep = NULL;
    bstring spec = Setting_get_str("control_port", &DEFAULT_CONTROL_SPEC);
    taskname("control");

    log_info("Setting up control socket in at %s", bdata(spec));

    CONTROL_SOCKET = mqsocket(ZMQ_REP);
    check(CONTROL_SOCKET != NULL, "Can't create contol socket.");

    rc = zmq_bind(CONTROL_SOCKET, bdata(spec));
    check(rc == 0, "Failed to bind control port to run/control.");

    while(CONTROL_RUNNING) {
        taskstate("waiting");

        req = read_message(CONTROL_SOCKET);
        check(req, "Failed to read message %s.",errno);

        rep = Control_execute(req);
        bdestroy(req);

        taskstate("replying");
        send_reply(CONTROL_SOCKET, rep);
        check(rep, "Faild to create a reply.");
    }

    rc = zmq_close(CONTROL_SOCKET);
    check(rc == 0, "Failed to close control port socket.");
    CONTROL_SOCKET = NULL;
    log_info("Control port exiting.");
    taskexit(0);

error:
    log_info("Control port exiting with error.");
    zmq_close(CONTROL_SOCKET);
    CONTROL_SOCKET = NULL;
    taskexit(1);
}

void Control_port_start()
{
    taskcreate(Control_task, NULL, 32 * 1024);
}

int Control_port_stop()
{
    int rc = 0;

    if(CONTROL_SOCKET != NULL) {
        rc = zmq_close(CONTROL_SOCKET);
        check(rc == 0, "Failed to close control socket");

        // TODO: wake up the control task.
        // it doesn't seem to notice that the socket has been closed.
    }

error:
    return rc;
}

#include "control.h"
#include "bstring.h"
#include "task/task.h"
#include "register.h"
#include "dbg.h"
#include <stdlib.h>
#include <time.h>
#include "setting.h"
#include <signal.h>

void bstring_free(void *data, void *hint)
{
    bdestroy((bstring)hint);
}

struct tagbstring REQ_STATUS_TASKS = bsStatic("status tasks");
struct tagbstring REQ_STATUS_NET = bsStatic("status net");

static inline bstring read_message(void *sock)
{
    bstring req = NULL;
    int rc = 0;

    zmq_msg_t *inmsg = calloc(sizeof(zmq_msg_t), 1);
    rc = zmq_msg_init(inmsg);
    check(rc == 0, "init failed.");

    rc = mqrecv(sock, inmsg, ZMQ_NOBLOCK);
    check(rc == 0, "Receive on control failed.");

    req = blk2bstr(zmq_msg_data(inmsg), zmq_msg_size(inmsg));
    check(req, "Failed to create the request string.");

    zmq_msg_close(inmsg);
    free(inmsg);

    return req;

error:
    if(inmsg) {
        zmq_msg_close(inmsg);
        free(inmsg);
    }

    return NULL;
}

static inline void send_reply(void *sock, bstring rep)
{
    int rc = 0;
    zmq_msg_t *outmsg = calloc(sizeof(zmq_msg_t), 1);
    rc = zmq_msg_init(outmsg);
    check(rc == 0, "init failed.");

    rc = zmq_msg_init_data(outmsg, bdata(rep), blength(rep), bstring_free, rep);
    check(rc == 0, "Failed to init reply data.");

    rc = mqsend(sock, outmsg, 0);
    check(rc == 0, "Failed to deliver 0mq message to requestor.");

error:
    free(outmsg);
    return;
}

static int CONTROL_RUNNING = 1;
static void *CONTROL_SOCKET = NULL;

%%{
    machine ControlParser;

    action mark { mark = fpc; }

    action status_tasks { reply = taskgetinfo(); fbreak; }
    action status_net { reply = Register_info(); fbreak; }

    action control_stop {
        reply = bfromcstr("{\"msg\": \"stopping control port\"}");
        CONTROL_RUNNING = 0; fbreak;
    }

    action time {
        reply = bformat("{\"time\": %d}", (int)time(NULL)); fbreak;
    }

    action kill {
        int id = atoi(p);

        if(id < 0 || id > MAX_REGISTERED_FDS) {
            reply = bformat("{\"error\": \"invalid id: %d\"}", id);
        } else {
            int fd = Register_fd_for_id(id);

            if(fd >= 0) {
                fdclose(fd);
                reply = bformat("{\"result\": \"killed %d\"}", id);
            } else {
                reply = bformat("{\"error\": \"does not exist: %d\"}", id);
            }
        }

        fbreak;
    }

    action reload {
        int rc = raise(SIGHUP);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be reloaded\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to reload the server\"}");
        }
        fbreak;
    }

    action stop {
        // TODO: probably report back the number of waiting tasks
        int rc = raise(SIGINT);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be stopped\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to stop the server\"}");
        }
        fbreak;
    }

    action terminate {
        // TODO: the server might have been terminated before
        // the reply is sent back. If this scenario is crucial (if possible at all)
        // find a workaround, otherwise, forget about it and remove this comment.
        int rc = raise(SIGTERM);
        if (0 == rc) {
            reply = bfromcstr("{\"msg\": \"the server will be terminated\"}");
        } else {
            reply = bfromcstr("{\"error\": \"failed to terminate the server\"}");
        }
        fbreak;
    }

    action help {
        reply = bfromcstr("{\"command_list\":[");
        bcatcstr(reply, "{\"name\": \"control stop\", \"description\": \"Close the control port.\"},\n");
        bcatcstr(reply, "{\"name\": \"kill <id>\", \"description\": \"Kill a connection in a violent way.\"},\n");
        bcatcstr(reply, "{\"name\": \"reload\", \"description\": \"Reload the server. Same as `kill -SIGHUP <server_pid>`.\"},\n");
        bcatcstr(reply, "{\"name\": \"status net\", \"description\": \"Return a list of active connections.\"},\n");
        bcatcstr(reply, "{\"name\": \"status tasks\", \"description\": \"Return a list of active tasks.\"},\n");
        bcatcstr(reply, "{\"name\": \"stop\", \"description\": \"Gracefully shut down the server. Same as `kill -SIGINT <server_pid>`.\"},\n");
        bcatcstr(reply, "{\"name\": \"terminate\", \"description\": \"Unconditionally terminate the server. Same as `kill -SIGTERM <server_pid>.\"},\n");
        bcatcstr(reply, "{\"name\": \"time\", \"description\": \"Return server timestamp.\"}\n");
        bcatcstr(reply, "]}\n");
        fbreak;
    }

    Status = "status" space+ ("tasks" @status_tasks | "net" @status_net);
    Control = "control stop" @control_stop;
    Time = "time" @time;
    Kill = "kill" space+ digit+ >mark @kill;
    Reload = "reload" @reload;
    Stop = "stop" @stop;
    Terminate = "terminate" @terminate;
    Help = "help" @help;

    main := Status | Control | Time | Kill | Reload | Stop | Terminate | Help;
}%%

%% write data;

bstring Control_execute(bstring req)
{
    const char *p = bdata(req);
    const char *pe = p+blength(req);
    const char *mark = NULL;
    int cs = 0;
    bstring reply = NULL;

    debug("RECEIVED CONTROL COMMAND: %s", bdata(req));

    %% write init;
    %% write exec noend;

    check(p <= pe, "Buffer overflow after parsing.  Tell Zed that you sent something from a handler that went %ld past the end in the parser.",
        (long int)(pe - p));

    if ( cs == %%{ write error; }%% ) {
        check(pe - p > 0, "Major erorr in the parser, tell Zed.");
        return bformat("{\"error\": \"parsing error at: ...%s\"}", bdata(req) + (pe - p));
    } else if(!reply) {
        return bformat("{\"error\": \"invalid command\"}", pe - p);
    } else {
        return reply;
    }

error:
    return bfromcstr("{\"error\": \"fatal error\"}");
}

struct tagbstring DEFAULT_CONTROL_SPEC = bsStatic("ipc://run/control");

void Control_task(void *v)
{
    int rc = 0;
    bstring req = NULL;
    bstring rep = NULL;
    bstring spec = Setting_get_str("control_port", &DEFAULT_CONTROL_SPEC);
    taskname("control");

    log_info("Setting up control socket in at %s", bdata(spec));

    CONTROL_SOCKET = mqsocket(ZMQ_REP);
    check(CONTROL_SOCKET != NULL, "Can't create contol socket.");

    rc = zmq_bind(CONTROL_SOCKET, bdata(spec));
    check(rc == 0, "Failed to bind control port to run/control.");

    while(CONTROL_RUNNING) {
        taskstate("waiting");

        req = read_message(CONTROL_SOCKET);
        check(req, "Failed to read message %s.",errno);

        rep = Control_execute(req);
        bdestroy(req);

        taskstate("replying");
        send_reply(CONTROL_SOCKET, rep);
        check(rep, "Faild to create a reply.");
    }

    rc = zmq_close(CONTROL_SOCKET);
    check(rc == 0, "Failed to close control port socket.");
    CONTROL_SOCKET = NULL;
    log_info("Control port exiting.");
    taskexit(0);

error:
    log_info("Control port exiting with error.");
    zmq_close(CONTROL_SOCKET);
    CONTROL_SOCKET = NULL;
    taskexit(1);
}

void Control_port_start()
{
    taskcreate(Control_task, NULL, 32 * 1024);
}

int Control_port_stop()
{
    int rc = 0;

    if(CONTROL_SOCKET != NULL) {
        rc = zmq_close(CONTROL_SOCKET);
        check(rc == 0, "Failed to close control socket");

        // TODO: wake up the control task.
        // it doesn't seem to notice that the socket has been closed.
    }

error:
    return rc;
}


improve formatting

improve formatting

%%{
# This ragel machine lexes hext. It is embedded in Parser::parse.
machine hext;

action error {
  this->throw_unexpected();
}

action act_newline {
  if( rule_start )
  {
    rule.consume_rule();
    rule_start = false;
  }
  else
  {
    // If a newline is encountered, reset indent, because the spaces could have
    // been used to indent a comment.
    rule.reset_indent();
  }
}

#### HELPERS ###################################################################
attr_name     = ( alpha (alnum | '-' | '_')** );
builtin_name  = ( alpha (alnum | '-' | '_')** );
cap_var       = ( [^/{}][^/{}]** );
match_literal = ( ( [^"] | '\\"' )** );
regex_content = ( ( [^/] | '\\/' )** );

#### NTH_PATTERN ###############################################################
# nth_pattern is used in some traits, e.g. nth-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  # nth-child(even)
  ( 'even' %{ pattern.set_nth_mul("2"); pattern.set_nth_add("0"); } )
  |

  # nth-child(odd)
  ( 'odd' %{ pattern.set_nth_mul("2"); pattern.set_nth_add("1"); } )
  |

  # nth-child(2n+1)
  ( ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.set_nth_mul(tok); } )
    ( 'n'    %{ pattern.set_nth_add("0"); }
      ( '+'
        ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.set_nth_add(tok); } )
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  # :empty
  ( 'empty' %{ pattern.consume_child_count("0"); } )
  |

  # :child-count(5)
  ( 'child-count('
    ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.consume_child_count(tok); } )
    ')' )
  |

  # :nth-child(2n+1)
  ( 'nth-child(' nth_pattern ')' %{ pattern.consume_nth_child(); } )
  |

  # :nth-last-child(2n+1)
  ( 'nth-last-child(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Back); } )
  |

  # :nth-of-type(2n+1)
  ( 'nth-of-type(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Front, rule.tag()); } )
  |

  # :first-child
  ( 'first-child' %{ pattern.set_nth_mul("1"); pattern.consume_nth_child(); } )
  |

  # :first-of-type
  ( 'first-of-type'
    %{ pattern.set_nth_mul("1");
       pattern.consume_nth_child(NthOff::Front, rule.tag()); } )
  |

  # :last-child
  ( 'last-child'
    %{ pattern.set_nth_mul("1"); pattern.consume_nth_child(NthOff::Back); } )
  |

  # :last-of-type
  ( 'last-of-type'
    %{ pattern.set_nth_mul("1");
       pattern.consume_nth_child(NthOff::Back, rule.tag()); } )
  |

  # :nth-last-of-type(2n+1)
  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Back, rule.tag()); } )
  |

  # :only-child
  ( 'only-child'
    %{ pattern.set_nth_mul("1"); pattern.consume_nth_child();
       pattern.set_nth_mul("1"); pattern.consume_nth_child(NthOff::Back); } )
);

# Literal value, e.g. attr="literal"
literal_value = '"' ( match_literal
  >{ TK_START; } %{ TK_STOP; pattern.set_attr_literal(tok); }
) '"';

# Regular expression
regex = '/' ( regex_content
    >{ TK_START; }
    %{ TK_STOP; pattern.set_regex_str(tok); }
  ) '/'
  # regex modifier:
  # 'i': case insensitive
  # 'c': collate (locale aware character groups)
  # '!': negate regex
  # Capture all characters to provide better error diagnostics.
  ( [a-zA-Z!]+ )?
  >{ TK_START; }
  %{ TK_STOP;
     if( !pattern.set_regex_mod(tok) )
       this->throw_unknown_token(tok, "regex modifier");
     try{ pattern.consume_regex(); }
     catch( const boost::regex_error& e )
     { // Mark whole regex as error, including slashes
       auto mark_len = pattern.regex_length() + tok.size() + 2;
       this->throw_regex_error(mark_len, e.code()); } };

#### ATTRIBUTES ################################################################
attributes = (
  ' '+
  (
    # attribute name
    (
      # builtin function, e.g. @text
      ( '@'
        ( builtin_name
          >{ TK_START; }
          %{ TK_STOP; { if( !pattern.set_builtin(tok) )
                        this->throw_unknown_token(tok, "builtin"); } } ) )
      |

      # html node attribute, e.g. class
      ( attr_name >{ TK_START; } %{ TK_STOP; pattern.set_attr_name(tok); } )
    )

    # attribute value
    (
      # literal operator, e.g. @text^="Lorem ", class~="menu"
      ( ( '^' | '*' | '!' | '~' | '$' ) >{ pattern.set_literal_op(*this->p); }
        '=' literal_value )
      |

      # capture variable, match regex or match literal
      ( '='
        (
          # capture variable, e.g. id={html_node_attr_id}, @text={heading}
          ( '{' cap_var >{ TK_START; }
                %{ TK_STOP;
                   if( rule.path() )
                     this->throw_error("Paths cannot have captures", tok.size());
                   pattern.set_cap_var(tok); }
                # optional capture regex, e.g. @text={time/(\d\d:\d\d)/}
                regex?
            '}' )
          |

          # match regex, e.g. id=/article_[0-9]+/
          regex
          |

          # match literal, e.g. id="article_23"
          literal_value
        )
      )
    )?
  ) %{ pattern.consume_pattern(); }
)+;

#### RULES #####################################################################
main := (
  # comments may be prefixed with any spaces and must terminate with a newline
  ( ' '* '#' (any - '\n')* ( '\n' %act_newline ) )
  |

  # rules
  (
    # groups of two spaces indent a rule
    ( '  ' %{ rule.increment_indent(); } )*

    # a rule starts with either '<' (direct descendant), '>' (any descendant)
    # or '~' (path)
    ( ( ( '<' )
        |
        ( '>' %{ rule.set_any_descendant(true); } )
        |
        ( '~' %{ rule.set_any_descendant(true); rule.set_path(true); } )
      ) %{ rule_start = true; }
    )

    # a rule can be optional, e.g. <?
    ( '?' %{ rule.set_optional(true); } )?

    # a rule can have a tag name, e.g. <div
    ( attr_name >{ TK_START; }
                %{ TK_STOP;
                   if( !rule.set_tag_name(tok) )
                     this->throw_unknown_token(tok, "html-tag"); }
    )?

    # a rule can have multiple traits, e.g. <div:first-child
    trait*

    # a rule can have multple attributes, e.g. <div class="menu"
    attributes?

    # a rule can be closed ("nodes may not have more attributes than specified")
    ( '>' %{ rule.set_closed(true); } )?
  )

  # rules and comments may terminate with a newline
  ( '\n' %act_newline )?
)** $err(error) $/act_newline $/{ fbreak; };

}%%

%%{
# This ragel machine lexes hext. It is embedded in Parser::parse.
machine hext;

action error {
  this->throw_unexpected();
}

action act_newline {
  if( rule_start )
  {
    rule.consume_rule();
    rule_start = false;
  }
  else
  {
    // If a newline is encountered, reset indent, because the spaces could have
    // been used to indent a comment.
    rule.reset_indent();
  }
}

#### HELPERS ###################################################################
attr_name     = ( alpha (alnum | '-' | '_')** );
builtin_name  = ( alpha (alnum | '-' | '_')** );
cap_var       = ( [^/{}][^/{}]** );
match_literal = ( ( [^"] | '\\"' )** );
regex_content = ( ( [^/] | '\\/' )** );

#### NTH_PATTERN ###############################################################
# nth_pattern is used in some traits, e.g. nth-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  # nth-child(even)
  ( 'even' %{ pattern.set_nth_mul("2"); pattern.set_nth_add("0"); } )
  |

  # nth-child(odd)
  ( 'odd' %{ pattern.set_nth_mul("2"); pattern.set_nth_add("1"); } )
  |

  # nth-child(2n+1)
  ( ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.set_nth_mul(tok); } )
    ( 'n'    %{ pattern.set_nth_add("0"); }
      ( '+'
        ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.set_nth_add(tok); } )
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  # :empty
  ( 'empty' %{ pattern.consume_child_count("0"); } )
  |

  # :child-count(5)
  ( 'child-count('
    ( [0-9]+ >{ TK_START; } %{ TK_STOP; pattern.consume_child_count(tok); } )
    ')' )
  |

  # :nth-child(2n+1)
  ( 'nth-child(' nth_pattern ')' %{ pattern.consume_nth_child(); } )
  |

  # :nth-last-child(2n+1)
  ( 'nth-last-child(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Back); } )
  |

  # :nth-of-type(2n+1)
  ( 'nth-of-type(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Front, rule.tag()); } )
  |

  # :first-child
  ( 'first-child' %{ pattern.set_nth_mul("1"); pattern.consume_nth_child(); } )
  |

  # :first-of-type
  ( 'first-of-type'
    %{ pattern.set_nth_mul("1");
       pattern.consume_nth_child(NthOff::Front, rule.tag()); } )
  |

  # :last-child
  ( 'last-child'
    %{ pattern.set_nth_mul("1"); pattern.consume_nth_child(NthOff::Back); } )
  |

  # :last-of-type
  ( 'last-of-type'
    %{ pattern.set_nth_mul("1");
       pattern.consume_nth_child(NthOff::Back, rule.tag()); } )
  |

  # :nth-last-of-type(2n+1)
  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pattern.consume_nth_child(NthOff::Back, rule.tag()); } )
  |

  # :only-child
  ( 'only-child'
    %{ pattern.set_nth_mul("1"); pattern.consume_nth_child();
       pattern.set_nth_mul("1"); pattern.consume_nth_child(NthOff::Back); } )
);

# attr="literal"
literal_value = '"' ( match_literal
  >{ TK_START; } %{ TK_STOP; pattern.set_attr_literal(tok); }
) '"';

# regular expression
regex = '/' ( regex_content
    >{ TK_START; }
    %{ TK_STOP; pattern.set_regex_str(tok); }
  ) '/'
  # regex modifier:
  # 'i': case insensitive
  # 'c': collate (locale aware character groups)
  # '!': negate regex
  # Capture all characters to provide better error diagnostics.
  ( [a-zA-Z!]+ )?
  >{ TK_START; }
  %{ TK_STOP;
     if( !pattern.set_regex_mod(tok) )
       this->throw_unknown_token(tok, "regex modifier");
     try{ pattern.consume_regex(); }
     catch( const boost::regex_error& e )
     { // Mark whole regex as error, including slashes
       auto mark_len = pattern.regex_length() + tok.size() + 2;
       this->throw_regex_error(mark_len, e.code()); } };

#### ATTRIBUTES ################################################################
attributes = (
  ' '+
  (
    # attribute name
    (
      # builtin function, e.g. @text
      ( '@'
        ( builtin_name
          >{ TK_START; }
          %{ TK_STOP; { if( !pattern.set_builtin(tok) )
                        this->throw_unknown_token(tok, "builtin"); } } ) )
      |

      # html node attribute, e.g. class
      ( attr_name >{ TK_START; } %{ TK_STOP; pattern.set_attr_name(tok); } )
    )

    # attribute value
    (
      # literal operator, e.g. @text^="Lorem ", class~="menu"
      ( ( '^' | '*' | '!' | '~' | '$' ) >{ pattern.set_literal_op(*this->p); }
        '=' literal_value )
      |

      # capture variable, match regex or match literal
      ( '='
        (
          # capture variable, e.g. id={html_node_attr_id}, @text={heading}
          ( '{' cap_var >{ TK_START; }
                %{ TK_STOP;
                   if( rule.path() )
                     this->throw_error("Paths cannot have captures", tok.size());
                   pattern.set_cap_var(tok); }
                # optional capture regex, e.g. @text={time/(\d\d:\d\d)/}
                regex?
            '}' )
          |

          # match regex, e.g. id=/article_[0-9]+/
          regex
          |

          # match literal, e.g. id="article_23"
          literal_value
        )
      )
    )?
  ) %{ pattern.consume_pattern(); }
)+;

#### RULES #####################################################################
main := (
  # comments may be prefixed with any spaces and must terminate with a newline
  ( ' '* '#' (any - '\n')* ( '\n' %act_newline ) )
  |

  (
    # groups of two spaces indent a rule
    ( '  ' %{ rule.increment_indent(); } )*

    # a rule starts with either '<' (direct descendant), '>' (any descendant)
    # or '~' (path)
    ( ( ( '<' )
        |
        ( '>' %{ rule.set_any_descendant(true); } )
        |
        ( '~' %{ rule.set_any_descendant(true); rule.set_path(true); } )
      ) %{ rule_start = true; }
    )

    # a rule can be optional, e.g. <?
    ( '?' %{ rule.set_optional(true); } )?

    # a rule can have a tag name, e.g. <div
    ( attr_name >{ TK_START; }
                %{ TK_STOP;
                   if( !rule.set_tag_name(tok) )
                     this->throw_unknown_token(tok, "html-tag"); }
    )?

    # a rule can have multiple traits, e.g. <div:first-child
    trait*

    # a rule can have multple attributes, e.g. <div class="menu"
    attributes?

    # a rule can be closed ("nodes may not have more attributes than specified")
    ( '>' %{ rule.set_closed(true); } )?
  )

  # rules and comments may terminate with a newline
  ( '\n' %act_newline )?
)** $err(error) $/act_newline $/{ fbreak; };

}%%


* remove test code

* remove test code

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: main.rl 2010-04-07 20:15:03 nineties $
 %);

include(stddef, code);

export main;

progname : NULL;

(% rename xxx.rl to xxx.p1 %);
change_suffix: (p0, p1) {
    allocate(3);
    x0 = strlen(p0);
    x1 = strlen(p1);
    x2 = memalloc(x0 - 2 + x1 + 1);
    memcpy(x2, p0, x0 - 2);
    memcpy(x2 + x0 - 2, p1, x1);
    wch(x2, x0 - 2 + x1, '\0');
    return x2;
};

(% p0: asm filename, p1: header filename, p2:rowl filename %);
compile: (p0, p1, p2) {
    allocate(4);

    init_typing();

    puts("compile ");
    puts(p2);
    putc('\n');

    (% lex and parse %);
    puts("> parsing...\n");
    x1 = parse(p2);

    (% type check %);
    puts("> typing...\n");
    x2 = typing(x1);

    (% generate header file %);
    puts("> generating header...\n");
    headergen(p1, x2);

    (% translate to two address code %);
    puts("> translating to tcode...\n");
    x3 = tcodegen(x2);

    (% generate assembly %);
    puts("> generating assembly...\n");
    x0 = open_out(p0);
    asmgen(x0, x3);
    close_out(x0);
    puts("finished\n");
};

ascmd: ["/usr/bin/as", NULL, "-o", NULL, NULL];
(% p0: object filename, p1: asm filename %);
assemble: (p0, p1) {
    allocate(1);
    x0 = fork(); (% pid %);
    if (x0 == 0) {
	(% child %);
	ascmd[1] = p1;
	ascmd[3] = p0;
	execve(ascmd[0], ascmd, 0);
    } else {
	waitpid(-1, &x1, 0);
    };
};

(% p0: argc, p1: argv %);
main: (p0, p1) {
    allocate(4);

    progname = strdup(p1[0]);

    x0 = 1;
    while (x0 < p0) {
	x1 = change_suffix(p1[x0], "s"); (% name of assembly source file %);
	x2 = change_suffix(p1[x0], "o"); (% name of object file %);
        x3 = change_suffix(p1[x0], "rli"); (% name of header file %);

	unlink(x1);
	unlink(x2);
        unlink(x3);

        compile(x1, x3, p1[x0]);
	assemble(x2, x1);
        x0 = x0 + 1;
    };
    return 0;
};

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: main.rl 2010-04-07 19:49:03 nineties $
 %);

include(stddef, code);

export main;

progname : NULL;

(% rename xxx.rl to xxx.p1 %);
change_suffix: (p0, p1) {
    allocate(3);
    x0 = strlen(p0);
    x1 = strlen(p1);
    x2 = memalloc(x0 - 2 + x1 + 1);
    memcpy(x2, p0, x0 - 2);
    memcpy(x2 + x0 - 2, p1, x1);
    wch(x2, x0 - 2 + x1, '\0');
    return x2;
};

(% p0: asm filename, p1: header filename, p2:rowl filename %);
compile: (p0, p1, p2) {
    allocate(4);

    init_typing();

    puts("compile ");
    puts(p2);
    putc('\n');

    (% lex and parse %);
    puts("> parsing...\n");
    x1 = parse(p2);

    (% type check %);
    puts("> typing...\n");
    x2 = typing(x1);

    (% generate header file %);
    puts("> generating header...\n");
    headergen(p1, x2);

    (% translate to two address code %);
    puts("> translating to tcode...\n");
    x3 = tcodegen(x2);

    (% generate assembly %);
    puts("> generating assembly...\n");
    x0 = open_out(p0);
    asmgen(x0, x3);
    close_out(x0);
    puts("finished\n");
};

ascmd: ["/usr/bin/as", NULL, "-o", NULL, NULL];
(% p0: object filename, p1: asm filename %);
assemble: (p0, p1) {
    allocate(1);
    x0 = fork(); (% pid %);
    if (x0 == 0) {
	(% child %);
	ascmd[1] = p1;
	ascmd[3] = p0;
	execve(ascmd[0], ascmd, 0);
    } else {
	waitpid(-1, &x1, 0);
    };
};

(% p0: argc, p1: argv %);
main: (p0, p1) {
    allocate(4);

    progname = strdup(p1[0]);

    x0 = 1;
    while (x0 < p0) {
	x1 = change_suffix(p1[x0], "s"); (% name of assembly source file %);
	x2 = change_suffix(p1[x0], "o"); (% name of object file %);
        x3 = change_suffix(p1[x0], "rli"); (% name of header file %);

	unlink(x1);
	unlink(x2);
        unlink(x3);

        compile(x1, x3, p1[x0]);
	assemble(x2, x1);
        x0 = x0 + 1;
    };
    return 0;
};

fib: (p0) {
    if (p0) {
        return p0 * fib(p0-1);
    };
    return 1;
};