Beginning table cleanup

Beginning table cleanup

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  cell_content = ^('|' | EOL);

  # table is one or more rows
  # a row is space* then any number of cells followed by a '| space* EOL'
  # a cell is a '|' followed by anything that is not a '|'

  Cell = cell_content+ >begin_content '|' >store_cell_content | '|' >no_content;

  table_row = space* '|' >initialize Cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);

  unclosed_table_row = space* '|' >initialize Cell* space* cell_content+ %/bad_table_row %/store_table :>> (EOL+ space*);

  Table = (table_row %store_row | unclosed_table_row >set_bad_table_row_line %bad_table_row)+ %store_table;

  main := Table;
}%%

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  cell_content = ^('|' | EOL);

  # table is one or more rows
  # a row is space* then any number of cells followed by a '| space* EOL'
  # a cell is a '|' followed by anything that is not a '|'

  cell = cell_content+ >begin_content '|' >store_cell_content | '|' >no_content;

  table_row = space* '|' >initialize cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);

  unclosed_table_row = space* '|' >initialize cell* space* cell_content+ %/bad_table_row %/store_table :>> (EOL+ space*);

  Table = (table_row %store_row | unclosed_table_row >set_bad_table_row_line %bad_table_row)+ %store_table;

  main := Table;
}%%


SSA: Ignore lines with the wrong number of fields.

git-svn-id: d8f21eb7283bfb39ebab2f8f9b4a259b233f9348@683 621663c8-3916-0410-8f58-edc14a8543d5

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div %d spans:",sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = -1;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = "[V4 Styles]" %ssaformat | "[V4+ Styles]" %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl (format %{eventformat=str;})? <: (event*);

		main := bom? header :> styles :> lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];
		div->wrapStyle = context->wrapStyle;

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" [0-9]+ "," [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) NSLog(@"parse error: %@",inputText);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	return divs;
}

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div %d spans:",sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = -1;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = "[V4 Styles]" %ssaformat | "[V4+ Styles]" %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl (format %{eventformat=str;})? <: (event*);

		main := bom? header :> styles :> lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];
		div->wrapStyle = context->wrapStyle;

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" [0-9]+ "," [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) NSLog(@"parse error: %@",inputText);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	return divs;
}


Allow whitespace between tag and identifier

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = [ \t]*;
	open  = '{' :> '{';
	close = '}' :> '}';

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/] | '-')+;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
	}

	#tag_type = [&/<>{^];

	tag = (
		open
		white
		identifier >start_identifier %got_identifier
		white
		close %mark
	) >write_static;

	text = (any+ -- open) ;

	main := (
		tag |
		text
	)* %eof(write_static);

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = [ \t]*;
	open  = '{' :> '{';
	close = '}' :> '}';

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/] | '-')+;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
	}

	#tag_type = [&/<>{^];

	tag = (
		open
		identifier >start_identifier %got_identifier
		close %mark
	) >write_static;

	text = (any+ -- open) ;

	main := (
		tag |
		text
	)* %eof(write_static);

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end


Allow multiple symbols in block statement

//
//  config_parser.rl
//

#include <cstdio>
#include <cassert>
#include <cstdlib>
#include <ctype.h>
#include <cstring>

#include "config_parser.h"

%%{

    machine config_parser;

    action mark             { mark = fpc; }
    action w_start_block    { start_block(); }
    action w_end_block      { end_block(); }
    action w_symbol         { symbol(mark, fpc - mark); }
    action w_end_statement  { end_statement(); }

    action done {
        config_done();
        fbreak;
    }

    Eol = ';' %w_end_statement;
    newline = ('\r' | '\n' ) | '\n';
    ws = (' ' | '\t' | '\r' | '\n' )+;
    comment = '/*' ( any* - ( any* '*/' any* ) ) '*/';
    symbol = ( ( any - ';' - ws - '{' - '}' )+ - ('/*') ) >mark %w_symbol;
    statement = ( symbol ( ws symbol)* ) ws* Eol;
    start_block = ( symbol ( ws symbol)* ) ws+ '{' %w_start_block;
    end_block = '}' ws* ';' %w_end_block;
    config = ( comment | start_block | end_block | statement | ws )* %done;

    main := config;

}%%

%% write data;

bool config_parser::parse(const char *buffer, size_t len)
{
    int cs = config_parser_en_main;

    const char *mark = NULL;
    const char *p = buffer;
    const char *pe = buffer + strlen(buffer);
    const char *eof = pe;

    %% write init;
    %% write exec;

    return (cs != config_parser_error && cs == config_parser_first_final);
}

//
//  config_parser.rl
//

#include <cstdio>
#include <cassert>
#include <cstdlib>
#include <ctype.h>
#include <cstring>

#include "config_parser.h"

%%{

    machine config_parser;

    action mark             { mark = fpc; }
    action w_start_block    { start_block(); }
    action w_end_block      { end_block(); }
    action w_symbol         { symbol(mark, fpc - mark); }
    action w_end_statement  { end_statement(); }

    action done {
        config_done();
        fbreak;
    }

    Eol = ';' %w_end_statement;
    newline = ('\r' | '\n' ) | '\n';
    ws = (' ' | '\t' | '\r' | '\n' )+;
    comment = '/*' ( any* - ( any* '*/' any* ) ) '*/';
    symbol = ( ( any - ';' - ws - '{' - '}' )+ - ('/*') ) >mark %w_symbol;
    statement = ( symbol ( ws symbol)* ) ws* Eol;
    start_block = symbol ws+ '{' %w_start_block;
    end_block = '}' ws* ';' %w_end_block;
    config = ( comment | start_block | end_block | statement | ws )* %done;

    main := config;

}%%

%% write data;

bool config_parser::parse(const char *buffer, size_t len)
{
    int cs = config_parser_en_main;

    const char *mark = NULL;
    const char *p = buffer;
    const char *pe = buffer + strlen(buffer);
    const char *eof = pe;

    %% write init;
    %% write exec;

    return (cs != config_parser_error && cs == config_parser_first_final);
}


Error handling

Error handling

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( ('ID=' identifier) | (key_word '=' value) ) @!{ error_code="key-value" };

  main := ( '##' ('FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( 'ID' = identifier | key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Fix off-by-1 location error.

Fix off-by-1 location error.

%%{

machine liquor;

whitespace  = [\t ]+;
symbol      = [a-zA-Z_];
any_newline = '\n' @ { line_starts.push(p + 1) } | any;
identifier  = symbol ( symbol | digit )*;

lblock  = '{%';
rblock  = '%}';

linterp = '{{';
rinterp = '}}';

lcomment = '{!';
rcomment = '!}';

action string_append {
  string << data[p]
}

action string_end {
  tok.(:string, string.dup, ts: str_start)
  string.clear
  fgoto code;
}

action runaway {
  runaway = true
}

action error {
  symbol = data[p]
  if symbol == "\n"
    symbol = "end of line"
  else
    symbol = "`#{symbol.inspect[1..-2]}'"
  end

  error = SyntaxError.new("unexpected #{symbol}",
    file:  name,
    line:  line_starts.count - 1,
    start: p - line_starts.last,
    end:   p - line_starts.last)
  raise error
}

comment := |*
    lcomment => { fcall comment; };
    rcomment => { fret; };
    any_newline;
*|;

dqstring := |*
    '\\"'  => { string << '"' };
    '\\\\' => { string << '\\' };
    '"'    => string_end;
    [^\n] @eof runaway
           => string_append;
    "\n"   => error;
*|;

sqstring := |*
    "\\'"  => { string << "'" };
    "\\\\" => { string << '\\' };
    "'"    => string_end;
    [^\n] @eof runaway
           => string_append;
    "\n"   => error;
*|;

integer := |*
    digit+  => { tok.(:integer, data[ts...te].to_i) };
    symbol  => error;
    any     => { fhold; fgoto code; };
*|;

tag_start := |*
    whitespace;

    'end ' identifier =>
      { tag = data[ts + 4...te]
        if tag_stack.last == tag
          fixtok.(:lblock2)
          tok.(:endtag)
          tag_stack.pop
          if registered_tags.include?(tag_stack.last)
            tag_conts = registered_tags[tag_stack.last].continuations
          end
        else
          (sl, sc), (el, ec) = loc.(ts), loc.(te - 1)
          info = { file: name, line: sl, start: sc, end: ec }
          if tag_stack.any?
            raise SyntaxError.new("unmatched `end #{tag}', expected `end #{tag_stack.last}'", info)
          else
            raise SyntaxError.new("unexpected `end #{tag}'", info)
          end
        end
        fgoto code;
      };

    identifier =>
      { tag = data[ts...te]

        if tag_conts.include? tag
          fixtok.(:lblock2)
          tok.(:keyword, tag)
          fgoto code;
        elsif registered_tags.include?(tag)
          tag_conts = registered_tags[tag].continuations
        end

        tok.(:ident, tag)
        last_tag = tag

        fgoto code;
      };

    identifier ':' =>
      { fixtok.(:lblock2)
        tok.(:keyword, data[ts...te - 1])
        fgoto code;
      };

    any =>
      { fhold; fgoto code; };
*|;

code := |*
    whitespace;

    identifier => { tok.(:ident, data[ts...te]) };

    digit => { fhold; fgoto integer; };

    identifier %{ kw_stop = p } ':' whitespace* rblock =>
      { tok.(:keyword, data[ts...kw_stop], te: kw_stop)
        tok.(:rblock,  nil,                ts: te - 2)
        if last_tag
          tag_stack.push last_tag
          last_tag = nil
        end
        fgoto plaintext;
      };

    identifier ':' =>
      { tok.(:keyword, data[ts...te-1]) };

    '=' whitespace* rblock =>
      {
        tok.(:keyword, '=', te: ts + 1)
        tok.(:rblock,  nil, ts: te - 2)
        if last_tag
          tag_stack.push last_tag
          last_tag = nil
        end
        fgoto plaintext;
      };

    '=' =>
      { tok.(:keyword, '=') };

    ','  => { tok.(:comma) };
    '.'  => { tok.(:dot)   };

    '['  => { tok.(:lbracket) };
    ']'  => { tok.(:rbracket) };

    '('  => { tok.(:lparen) };
    ')'  => { tok.(:rparen) };

    '|'  => { tok.(:pipe) };

    '+'  => { tok.(:op_plus)  };
    '-'  => { tok.(:op_minus) };
    '*'  => { tok.(:op_mul)   };
    '/'  => { tok.(:op_div)   };
    '%'  => { tok.(:op_mod)   };

    '==' => { tok.(:op_eq)  };
    '!=' => { tok.(:op_neq) };
    '>'  => { tok.(:op_gt)  };
    '>=' => { tok.(:op_geq) };
    '<'  => { tok.(:op_lt)  };
    '<=' => { tok.(:op_leq) };

    '!'  => { tok.(:op_not) };

    '&&' => { tok.(:op_and) };
    '||' => { tok.(:op_or) };

    '"'  => { str_start = p; fgoto dqstring; };
    "'"  => { str_start = p; fgoto sqstring; };

    rinterp => { tok.(:rinterp); fgoto plaintext; };
    rblock  => { tok.(:rblock);  fgoto plaintext; };

    any => error;
*|;

plaintext := |*
    ( '\\{' [%{!]? | '{' [^%{!] | any_newline - '{' )* =>
      { tok.(:plaintext, data[ts...te]); };

    '{{' =>
      { tok.(:linterp); fgoto code; };

    '{%' =>
      { tok.(:lblock);  fgoto tag_start; };

    '{!' =>
      { fcall comment; };

    any_newline =>
      { fhold; fgoto code; };
*|;

}%%

module Liquor
  module Lexer
    %% write data;

    def self.lex(data, name='(code)', registered_tags={})
      eof    = data.length
      ts     = nil # token start
      te     = nil # token end
      stack  = []

      # Strings
      string    = ""
      str_start = nil
      runaway   = false

      # Tags
      tag_stack  = []
      tag_conts  = []
      last_tag   = nil
      kw_stop    = nil

      line_starts = [0]

      find_line_start = ->(index) {
        line_starts.
            each_index.find { |i|
              line_starts[i + 1].nil? || line_starts[i + 1] > index
            }
      }
      loc = ->(index) {
        line_start_index = find_line_start.(index)
        [ line_start_index, index - line_starts[line_start_index] ]
      }

      tokens = []

      fixtok = ->(new_type) {
        tokens.last[0] = new_type
      }
      tok = ->(type, data=nil, options={}) {
        sl, sc, el, ec = *loc.(options[:ts] || ts),
                         *loc.(options[:te] || te - 1)
        tokens << [type, { file: name, line: sl, start: sc, end: ec }, *data]
      }

      %% write init;
      %% write exec;

      if runaway
        line_start_index = find_line_start.(str_start)
        line_start = line_starts[line_start_index]

        error = SyntaxError.new("literal not terminated",
          file:  name,
          line:  line_start_index,
          start: str_start - line_start,
          end:   str_start - line_start)
        raise error
      end

      tokens
    end
  end
end

%%{

machine liquor;

whitespace  = [\t ]+;
symbol      = [a-zA-Z_];
any_newline = '\n' @ { line_starts.push(p + 1) } | any;
identifier  = symbol ( symbol | digit )*;

lblock  = '{%';
rblock  = '%}';

linterp = '{{';
rinterp = '}}';

lcomment = '{!';
rcomment = '!}';

action string_append {
  string << data[p]
}

action string_end {
  tok.(:string, string.dup, ts: str_start)
  string.clear
  fgoto code;
}

action runaway {
  runaway = true
}

action error {
  symbol = data[p]
  if symbol == "\n"
    symbol = "end of line"
  else
    symbol = "`#{symbol.inspect[1..-2]}'"
  end

  error = SyntaxError.new("unexpected #{symbol}",
    file:  name,
    line:  line_starts.count - 1,
    start: p - line_starts.last,
    end:   p - line_starts.last)
  raise error
}

comment := |*
    lcomment => { fcall comment; };
    rcomment => { fret; };
    any_newline;
*|;

dqstring := |*
    '\\"'  => { string << '"' };
    '\\\\' => { string << '\\' };
    '"'    => string_end;
    [^\n] @eof runaway
           => string_append;
    "\n"   => error;
*|;

sqstring := |*
    "\\'"  => { string << "'" };
    "\\\\" => { string << '\\' };
    "'"    => string_end;
    [^\n] @eof runaway
           => string_append;
    "\n"   => error;
*|;

integer := |*
    digit+  => { tok.(:integer, data[ts...te].to_i) };
    symbol  => error;
    any     => { fhold; fgoto code; };
*|;

tag_start := |*
    whitespace;

    'end ' identifier =>
      { tag = data[ts + 4...te]
        if tag_stack.last == tag
          fixtok.(:lblock2)
          tok.(:endtag)
          tag_stack.pop
          if registered_tags.include?(tag_stack.last)
            tag_conts = registered_tags[tag_stack.last].continuations
          end
        else
          (sl, sc), (el, ec) = loc.(ts), loc.(te)
          info = { file: name, line: sl, start: sc, end: ec }
          if tag_stack.any?
            raise SyntaxError.new("unmatched `end #{tag}', expected `end #{tag_stack.last}'", info)
          else
            raise SyntaxError.new("unexpected `end #{tag}'", info)
          end
        end
        fgoto code;
      };

    identifier =>
      { tag = data[ts...te]

        if tag_conts.include? tag
          fixtok.(:lblock2)
          tok.(:keyword, tag)
          fgoto code;
        elsif registered_tags.include?(tag)
          tag_conts = registered_tags[tag].continuations
        end

        tok.(:ident, tag)
        last_tag = tag

        fgoto code;
      };

    identifier ':' =>
      { fixtok.(:lblock2)
        tok.(:keyword, data[ts...te - 1])
        fgoto code;
      };

    any =>
      { fhold; fgoto code; };
*|;

code := |*
    whitespace;

    identifier => { tok.(:ident, data[ts...te]) };

    digit => { fhold; fgoto integer; };

    identifier %{ kw_stop = p } ':' whitespace* rblock =>
      { tok.(:keyword, data[ts...kw_stop], te: kw_stop)
        tok.(:rblock,  nil,                ts: te - 2)
        if last_tag
          tag_stack.push last_tag
          last_tag = nil
        end
        fgoto plaintext;
      };

    identifier ':' =>
      { tok.(:keyword, data[ts...te-1]) };

    '=' whitespace* rblock =>
      {
        tok.(:keyword, '=', te: ts + 1)
        tok.(:rblock,  nil, ts: te - 2)
        if last_tag
          tag_stack.push last_tag
          last_tag = nil
        end
        fgoto plaintext;
      };

    '=' =>
      { tok.(:keyword, '=') };

    ','  => { tok.(:comma) };
    '.'  => { tok.(:dot)   };

    '['  => { tok.(:lbracket) };
    ']'  => { tok.(:rbracket) };

    '('  => { tok.(:lparen) };
    ')'  => { tok.(:rparen) };

    '|'  => { tok.(:pipe) };

    '+'  => { tok.(:op_plus)  };
    '-'  => { tok.(:op_minus) };
    '*'  => { tok.(:op_mul)   };
    '/'  => { tok.(:op_div)   };
    '%'  => { tok.(:op_mod)   };

    '==' => { tok.(:op_eq)  };
    '!=' => { tok.(:op_neq) };
    '>'  => { tok.(:op_gt)  };
    '>=' => { tok.(:op_geq) };
    '<'  => { tok.(:op_lt)  };
    '<=' => { tok.(:op_leq) };

    '!'  => { tok.(:op_not) };

    '&&' => { tok.(:op_and) };
    '||' => { tok.(:op_or) };

    '"'  => { str_start = p; fgoto dqstring; };
    "'"  => { str_start = p; fgoto sqstring; };

    rinterp => { tok.(:rinterp); fgoto plaintext; };
    rblock  => { tok.(:rblock);  fgoto plaintext; };

    any => error;
*|;

plaintext := |*
    ( '\\{' [%{!]? | '{' [^%{!] | any_newline - '{' )* =>
      { tok.(:plaintext, data[ts...te]); };

    '{{' =>
      { tok.(:linterp); fgoto code; };

    '{%' =>
      { tok.(:lblock);  fgoto tag_start; };

    '{!' =>
      { fcall comment; };

    any_newline =>
      { fhold; fgoto code; };
*|;

}%%

module Liquor
  module Lexer
    %% write data;

    def self.lex(data, name='(code)', registered_tags={})
      eof    = data.length
      ts     = nil # token start
      te     = nil # token end
      stack  = []

      # Strings
      string    = ""
      str_start = nil
      runaway   = false

      # Tags
      tag_stack  = []
      tag_conts  = []
      last_tag   = nil
      kw_stop    = nil

      line_starts = [0]

      find_line_start = ->(index) {
        line_starts.
            each_index.find { |i|
              line_starts[i + 1].nil? || line_starts[i + 1] > index
            }
      }
      loc = ->(index) {
        line_start_index = find_line_start.(index)
        [ line_start_index, index - line_starts[line_start_index] ]
      }

      tokens = []

      fixtok = ->(new_type) {
        tokens.last[0] = new_type
      }
      tok = ->(type, data=nil, options={}) {
        sl, sc, el, ec = *loc.(options[:ts] || ts),
                         *loc.(options[:te] || te - 1)
        tokens << [type, { file: name, line: sl, start: sc, end: ec }, *data]
      }

      %% write init;
      %% write exec;

      if runaway
        line_start_index = find_line_start.(str_start)
        line_start = line_starts[line_start_index]

        error = SyntaxError.new("literal not terminated",
          file:  name,
          line:  line_start_index,
          start: str_start - line_start,
          end:   str_start - line_start)
        raise error
      end

      tokens
    end
  end
end


fixed warning in mailbox2.rl test case

fixed warning in mailbox2.rl test case

/*
 * @LANG: c++
 * @CFLAGS: -I../aapl
 */

#include <iostream>
#include <string.h>

using std::cin;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine mailbox;

	action prn_char { cout << *p; }
	action prn_space { cout << ' '; }
	action prn_word { cout.write(ws, p-ws); cout << ' '; }
	action prn_addr1 { cout << "| "; cout.write(ws+1, p-ws-2); }
	action prn_addr2 { cout << "| "; cout.write(ws, p-ws); }
	action prn_tab { cout << '\t'; }
	action prn_nl { cout << '\n'; }
	action prn_separator { cout << "------\n"; }
	action prn_from { cout << "FROM\n"; }
	action prn_to { cout << "TO\n"; }
	action prn_subj { cout << "SUBJECT\n"; }

	action start_word { ws = p; }
	action start_headers { }
	action end_headers { }

	day = upper lower{2};
	month = upper lower{2};
	year = digit{4};
	time = digit{2} ':' digit{2}
		( ':' digit{2} )?;
	letterZone = upper{3};
	numZone = [+\-] digit{4};
	zone = letterZone | numZone;
	dayNum = ( digit | ' ' ) digit;

	date = day ' ' month ' '
		dayNum ' ' time ' '
		(
			year |
			year ' ' zone |
			zone ' ' year
		);

	fromLine = 'From ' [^\n]* ' '
		date '\n' @start_headers;

	headerChar = print - [ :];
	headersToPrint = 'From' |
		'To' | 'Subject';
	headersToConsume =
		headerChar+ - headersToPrint;

	consumeHeader =
		headersToConsume ':'
		(
			[^\n] |
			( '\n' [ \t] )
		)*
		'\n';

	addrWS = ( [ \t]+ | '\n' [ \t]+ );
	addrComment = '(' [^)]* ')';
	addrWord = [^"'@,<>() \t\n]+;
	addrAddr1 = '<' [^>]* '>';
	addrAddr2 = addrWord '@' addrWord;
	addrString =
		'"' [^"]* '"' |
		"'" [^']* "'";

	addrItem = (
			addrAddr1 %prn_addr1 |
			addrAddr2 %prn_addr2 |
			addrWord %prn_word |
			addrString %prn_word
		) >start_word;

	address = (
			addrWS |
			addrComment |
			addrItem
		)** >prn_tab;

	addrHeader = (
			'From' %prn_from |
			'To' %prn_to
		) ':'
		address ( ',' @prn_nl address )*
		'\n' %prn_nl;

	subjectHeader =
		'Subject:' @prn_subj @prn_tab
		' '* <:
		(
			[^\n] @prn_char  |
			( '\n' [ \t]+ ) %prn_space
		)**
		'\n' %prn_nl;

	header = consumeHeader |
		addrHeader | subjectHeader;

	messageLine =
		( [^\n]* '\n' - fromLine );

	main := (
			fromLine %prn_separator
			header*
			'\n' @end_headers
			messageLine*
		)*;
 }%%

%% write data;

#define BUFSIZE 8192

void test( const char *buf )
{
	int cs, len = strlen( buf );
	const char *ws = 0;

	%% write init;
	const char *p = buf;
	const char *pe = p + len;
	%% write exec;

	if ( cs == mailbox_error )
		cerr << "ERROR" << endl;

	if ( cs < mailbox_first_final )
		cerr << "DID NOT FINISH IN A FINAL STATE" << endl;
}

int main()
{
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"Message goes here. \n"
		"From (trick from line).\n"
		"From: not really a header\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) \n"
		"\n"
		"Message goes here. \n"
		"\n"
	);
	return 0;
}

#ifdef _____OUTPUT_____
------
FROM
	"Adrian D. Thurston" | thurston@complang.org
SUBJECT
	the squirrel has landed
------
TO
	Edgar Allen Poe | ep@net.com
SUBJECT
	(no subject)
#endif

/*
 * @LANG: c++
 * @CFLAGS: -I../aapl
 */

#include <iostream>
#include <string.h>

using std::cin;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine mailbox;

	action prn_char { cout << *p; }
	action prn_space { cout << ' '; }
	action prn_word { cout.write(ws, p-ws); cout << ' '; }
	action prn_addr1 { cout << "| "; cout.write(ws+1, p-ws-2); }
	action prn_addr2 { cout << "| "; cout.write(ws, p-ws); }
	action prn_tab { cout << '\t'; }
	action prn_nl { cout << '\n'; }
	action prn_separator { cout << "------\n"; }
	action prn_from { cout << "FROM\n"; }
	action prn_to { cout << "TO\n"; }
	action prn_subj { cout << "SUBJECT\n"; }

	action start_word { ws = p; }
	action start_headers { preserve = p; }
	action end_headers {preserve = 0;}

	day = upper lower{2};
	month = upper lower{2};
	year = digit{4};
	time = digit{2} ':' digit{2}
		( ':' digit{2} )?;
	letterZone = upper{3};
	numZone = [+\-] digit{4};
	zone = letterZone | numZone;
	dayNum = ( digit | ' ' ) digit;

	date = day ' ' month ' '
		dayNum ' ' time ' '
		(
			year |
			year ' ' zone |
			zone ' ' year
		);

	fromLine = 'From ' [^\n]* ' '
		date '\n' @start_headers;

	headerChar = print - [ :];
	headersToPrint = 'From' |
		'To' | 'Subject';
	headersToConsume =
		headerChar+ - headersToPrint;

	consumeHeader =
		headersToConsume ':'
		(
			[^\n] |
			( '\n' [ \t] )
		)*
		'\n';

	addrWS = ( [ \t]+ | '\n' [ \t]+ );
	addrComment = '(' [^)]* ')';
	addrWord = [^"'@,<>() \t\n]+;
	addrAddr1 = '<' [^>]* '>';
	addrAddr2 = addrWord '@' addrWord;
	addrString =
		'"' [^"]* '"' |
		"'" [^']* "'";

	addrItem = (
			addrAddr1 %prn_addr1 |
			addrAddr2 %prn_addr2 |
			addrWord %prn_word |
			addrString %prn_word
		) >start_word;

	address = (
			addrWS |
			addrComment |
			addrItem
		)** >prn_tab;

	addrHeader = (
			'From' %prn_from |
			'To' %prn_to
		) ':'
		address ( ',' @prn_nl address )*
		'\n' %prn_nl;

	subjectHeader =
		'Subject:' @prn_subj @prn_tab
		' '* <:
		(
			[^\n] @prn_char  |
			( '\n' [ \t]+ ) %prn_space
		)**
		'\n' %prn_nl;

	header = consumeHeader |
		addrHeader | subjectHeader;

	messageLine =
		( [^\n]* '\n' - fromLine );

	main := (
			fromLine %prn_separator
			header*
			'\n' @end_headers
			messageLine*
		)*;
 }%%

%% write data;

#define BUFSIZE 8192

void test( const char *buf )
{
	int cs, len = strlen( buf );
	const char *preserve = 0, *ws = 0;

	%% write init;
	const char *p = buf;
	const char *pe = p + len;
	%% write exec;

	if ( cs == mailbox_error )
		cerr << "ERROR" << endl;

	if ( cs < mailbox_first_final )
		cerr << "DID NOT FINISH IN A FINAL STATE" << endl;
}

int main()
{
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"Message goes here. \n"
		"From (trick from line).\n"
		"From: not really a header\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) \n"
		"\n"
		"Message goes here. \n"
		"\n"
	);
	return 0;
}

#ifdef _____OUTPUT_____
------
FROM
	"Adrian D. Thurston" | thurston@complang.org
SUBJECT
	the squirrel has landed
------
TO
	Edgar Allen Poe | ep@net.com
SUBJECT
	(no subject)
#endif