SSA: Don't call the parser actions for tags no renderer supports ATM.

/*
 * SubParsing.m.rl
 * Created by Alexander Strange on 7/25/07.
 *
 * This file is part of Perian.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize
{
	[delegate releaseSpanExtra:ex];
	[super finalize];
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"Span at %d: %@", offset, [delegate describeSpanEx:ex]];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(NSString *ssastr, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	unsigned len = [ssastr length];
	NSData *ssaData;
	const unichar *ssa = STUnicodeForString(ssastr, &ssaData);
	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

#define send() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)strbegin length:p-strbegin freeWhenDone:NO] autorelease]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl :> (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend %/csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl :> (format %{eventformat=str;})? <: (event*);

		main := bom? header styles lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	[ssaData release];

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = *(id*)a, *divB = *(id*)b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

#undef send
#define send()  [[[NSString alloc] initWithCharactersNoCopy:(unichar*)outputbegin length:p-outputbegin freeWhenDone:NO] autorelease]
#define psend() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)parambegin length:p-parambegin freeWhenDone:NO] autorelease]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			size_t linelen = [inputText length];
			NSData *linebufData;
			const unichar *linebuf = STUnicodeForString(inputText, &linebufData);
			const unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_tag_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; float floatnum = 0;
			NSString *strval=NULL;
			float curX, curY;
			int intnum = 0;
			BOOL reachedEnd = NO, setWrapStyle = NO, setPosition = NO, setAlignForDiv = NO, dropThisSpan = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}
				action drawingmode {tag(p, floatnum); dropThisSpan = floatnum > 0;}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action nullstring {strval = @"";}
				action setpos {curX=curY=0; sscanf([psend() UTF8String], "(%f,%f", &curX, &curY);}

				action ssaalign {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
					}
				}

				action align {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(intnum, &div->alignH, &div->alignV);
					}
				}

				action wrapstyle {
					if (!setWrapStyle) {
						setWrapStyle = YES;

						div->wrapStyle = intnum;

					}
				}

				action position {
					if (!setPosition) {
						setPosition = YES;

						div->posX = curX;
						div->posY = curY;
						div->positioned = YES;
					}
				}

				action origin {
					div->shouldResetPens = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatn = ("-"? ([0-9]+ ("." [0-9]*)?) | ([0-9]* "." [0-9]+));
				floatnum = floatn >paramset %setfloatnum;
				string = (([^\\}]+) >paramset %setstringval | "" %nullstring );
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				pos = ("(" floatn "," floatn ")") >paramset %setpos;
				move = ("(" (floatn ","){3,5} floatn ")") >paramset %setpos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" floatnum
							|"blur" floatnum
							|"fax" floatnum
							|"fay" floatnum
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum
							|"fry" floatnum
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" pos %position
							|"move" move %position
							|"t" parens
							|"org" parens %origin
							|("fad" "e"? parens)
							|"i"? "clip" parens
							|"p" floatnum %drawingmode
							|"pbo" floatnum
							|"xbord" floatnum
							|"ybord" floatnum
							|"xshad" floatnum
							|"yshad" floatnum
					   );

				tag = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					p--;
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
						}

						[div->text appendFormat:@"%C",o];
					}

					chars_deleted++;

					p++;
					outputbegin = p+1;
				}

				action enter_tag {
					if (dropThisSpan) chars_deleted += p - outputbegin;
					else if (p > outputbegin) [div->text appendString:send()];
					if (p == pe) reachedEnd = YES;

					if (p != pb) {
						[div->spans addObject:current_span];

						if (!reachedEnd) current_span = [current_span cloneWithDelegate:delegate];
					}

					last_tag_start = p;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_tag_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					p--;
				}

				special = ("\\" :> any) @backslash_handler | tag >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char+;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reachedEnd) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[linebufData release];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}

/*
 * SubParsing.m.rl
 * Created by Alexander Strange on 7/25/07.
 *
 * This file is part of Perian.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize
{
	[delegate releaseSpanExtra:ex];
	[super finalize];
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"Span at %d: %@", offset, [delegate describeSpanEx:ex]];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(NSString *ssastr, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	unsigned len = [ssastr length];
	NSData *ssaData;
	const unichar *ssa = STUnicodeForString(ssastr, &ssaData);
	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

#define send() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)strbegin length:p-strbegin freeWhenDone:NO] autorelease]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl :> (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend %/csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl :> (format %{eventformat=str;})? <: (event*);

		main := bom? header styles lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	[ssaData release];

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = *(id*)a, *divB = *(id*)b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

#undef send
#define send()  [[[NSString alloc] initWithCharactersNoCopy:(unichar*)outputbegin length:p-outputbegin freeWhenDone:NO] autorelease]
#define psend() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)parambegin length:p-parambegin freeWhenDone:NO] autorelease]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			size_t linelen = [inputText length];
			NSData *linebufData;
			const unichar *linebuf = STUnicodeForString(inputText, &linebufData);
			const unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_tag_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; float floatnum = 0;
			NSString *strval=NULL;
			float curX, curY;
			int intnum = 0;
			BOOL reachedEnd = NO, setWrapStyle = NO, setPosition = NO, setAlignForDiv = NO, dropThisSpan = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}
				action drawingmode {tag(p, floatnum); dropThisSpan = floatnum > 0;}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action nullstring {strval = @"";}
				action setpos {curX=curY=0; sscanf([psend() UTF8String], "(%f,%f", &curX, &curY);}

				action ssaalign {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
					}
				}

				action align {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(intnum, &div->alignH, &div->alignV);
					}
				}

				action wrapstyle {
					if (!setWrapStyle) {
						setWrapStyle = YES;

						div->wrapStyle = intnum;

					}
				}

				action position {
					if (!setPosition) {
						setPosition = YES;

						div->posX = curX;
						div->posY = curY;
						div->positioned = YES;
					}
				}

				action origin {
					div->shouldResetPens = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatn = ("-"? ([0-9]+ ("." [0-9]*)?) | ([0-9]* "." [0-9]+));
				floatnum = floatn >paramset %setfloatnum;
				string = (([^\\}]+) >paramset %setstringval | "" %nullstring );
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				pos = ("(" floatn "," floatn ")") >paramset %setpos;
				move = ("(" (floatn ","){3,5} floatn ")") >paramset %setpos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" floatnum %bluredge
							|"blur" floatnum
							|"fax" floatnum
							|"fay" floatnum
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" pos %position
							|"move" move %position
							|"t" parens
							|"org" parens %origin
							|("fad" "e"? parens)
							|"i"? "clip" parens
							|"p" floatnum %drawingmode
							|"pbo" floatnum
							|"xbord" floatnum
							|"ybord" floatnum
							|"xshad" floatnum
							|"yshad" floatnum
					   );

				tag = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					p--;
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
						}

						[div->text appendFormat:@"%C",o];
					}

					chars_deleted++;

					p++;
					outputbegin = p+1;
				}

				action enter_tag {
					if (dropThisSpan) chars_deleted += p - outputbegin;
					else if (p > outputbegin) [div->text appendString:send()];
					if (p == pe) reachedEnd = YES;

					if (p != pb) {
						[div->spans addObject:current_span];

						if (!reachedEnd) current_span = [current_span cloneWithDelegate:delegate];
					}

					last_tag_start = p;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_tag_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					p--;
				}

				special = ("\\" :> any) @backslash_handler | tag >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char+;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reachedEnd) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[linebufData release];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}


Reformatted state machine specification.

In particular:

- fit specifications to fit within 80 columns
- organized blocks into rough categories

require 'castanet'

%%{
  machine parser;

  # Actions
  # -------

  action buffer { buffer << fc }
  action saveUsername { r.username = buffer; buffer = '' }
  action saveFailureCode { r.failure_code = buffer; buffer = '' }
  action saveFailureReason { r.failure_reason = buffer.strip; buffer = '' }
  action setAuthenticated { r.authenticated = true; eof = -1 }

  # XML definitions
  # ---------------

  quote       = '"' | "'";
  xmlContent  = any -- [<&];

  # Leaf tags
  # ---------

  code    = ( ( upper | '_' ) @buffer )+ %saveFailureCode;
  reason  = ( xmlContent @buffer )+ %saveFailureReason;
  user    = "<cas:user>" ( xmlContent @buffer )+ %saveUsername "</cas:user>";

  # Non-leaf tags
  # -------------

  serviceResponseStart         = "<cas:serviceResponse xmlns:cas="
                                 quote
                                 "http://www.yale.edu/tp/cas"
                                 quote
                                 ">";
  serviceResponseEnd           = "</cas:serviceResponse>";

  authenticationFailureStart   = "<cas:authenticationFailure code="
                                 quote
                                 code
                                 quote
                                 ">";
  authenticationFailureEnd     = "</cas:authenticationFailure>";

  authenticationSuccessStart   = "<cas:authenticationSuccess>";
  authenticationSuccessEnd     = "</cas:authenticationSuccess>";

  # Top-level elements
  # ------------------

  ok_cas_st     = ( serviceResponseStart
                    space*
                    authenticationSuccessStart
                    space*
                    user
                    space*
                    authenticationSuccessEnd
                    space*
                    serviceResponseEnd ) @setAuthenticated;

  failed_cas_st = ( serviceResponseStart
                    space*
                    authenticationFailureStart
                    reason
                    authenticationFailureEnd
                    space*
                    serviceResponseEnd );

  cas_st        = ok_cas_st | failed_cas_st;

  main := cas_st;
}%%

module Castanet
  class Response
    ##
    # Whether or not this response passed CAS authentication.
    #
    # @return [Boolean]
    attr_accessor :authenticated

    alias_method :authenticated?, :authenticated

    ##
    # The failure code returned on authentication failure.
    #
    # @return [String, nil]
    attr_accessor :failure_code

    ##
    # The reason given by the CAS server for authentication failure.
    #
    # @return [String, nil]
    attr_accessor :failure_reason

    ##
    # The name of the owner of the validated service or proxy ticket.
    #
    # This information is only present on authentication success.
    #
    # @return [String, nil]
    attr_accessor :username

    ##
    # Generates a {Response} object from a CAS response.
    #
    # @param [String] response the CAS response
    # @return [Response]
    def self.from_cas(response)
      data = response.strip.unpack('c*')
      buffer = ''
      eof = nil

      %% write init;

      new.tap do |r|
        %% write exec;
      end
    end

    def initialize
      self.authenticated = false
    end

    %% write data;
  end
end

require 'castanet'

%%{
  machine parser;

  action buffer { buffer << data.slice(p, 1).pack('c') }
  action saveUsername { r.username = buffer; buffer = '' }
  action saveFailureCode { r.failure_code = buffer; buffer = '' }
  action saveFailureReason { r.failure_reason = buffer.strip; buffer = '' }

  quote = '"' | "'";
  xmlContent = any -- [<&];

  serviceResponseStart         = "<cas:serviceResponse xmlns:cas=" quote "http://www.yale.edu/tp/cas" quote ">";

  code = ( ( upper | '_' ) @buffer )+ %saveFailureCode;
  reason = ( xmlContent @buffer )+ %saveFailureReason;

  authenticationFailureStart   = "<cas:authenticationFailure code=" quote code quote ">";
  authenticationSuccessStart   = "<cas:authenticationSuccess>";

  user                         = "<cas:user>" ( xmlContent @buffer )+ %saveUsername "</cas:user>";

  authenticationFailureEnd     = "</cas:authenticationFailure>";
  authenticationSuccessEnd     = "</cas:authenticationSuccess>";
  serviceResponseEnd           = "</cas:serviceResponse>";

  action setAuthenticated { r.authenticated = true; eof = -1 }

  ok_cas_st = ( serviceResponseStart space* authenticationSuccessStart space* user space* authenticationSuccessEnd space* serviceResponseEnd ) @setAuthenticated;
  failed_cas_st = ( serviceResponseStart space* authenticationFailureStart reason authenticationFailureEnd space* serviceResponseEnd );
  cas_st = ok_cas_st | failed_cas_st;

  main := cas_st;
}%%

module Castanet
  class Response
    ##
    # Whether or not this response passed CAS authentication.
    #
    # @return [Boolean]
    attr_accessor :authenticated

    alias_method :authenticated?, :authenticated

    ##
    # The failure code returned on authentication failure.
    #
    # @return [String, nil]
    attr_accessor :failure_code

    ##
    # The reason given by the CAS server for authentication failure.
    #
    # @return [String, nil]
    attr_accessor :failure_reason

    ##
    # The name of the owner of the validated service or proxy ticket.
    #
    # This information is only present on authentication success.
    #
    # @return [String, nil]
    attr_accessor :username

    ##
    # Generates a {Response} object from a CAS response.
    #
    # @param [String] response the CAS response
    # @return [Response]
    def self.from_cas(response)
      data = response.strip.unpack('c*')
      buffer = ''
      eof = nil

      %% write init;

      new.tap do |r|
        %% write exec;
      end
    end

    def initialize
      self.authenticated = false
    end

    %% write data;
  end
end


add space after comma in typized argument lists

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ('<' . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


Cleaned up lexing of comments/cdata.

Cleaned up lexing of comments/cdata.

Thanks to @whitequark for suggesting the use of the "--" operator.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback("on_comment", data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback("on_cdata", data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote+ dquote);
    string_squote = (squote ^squote+ squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #
    action start_element {
        callback_simple("on_element_start");
        fnext element_name;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;

        # The start of an element.
        '<' => start_element;

        # Regular closing tags.
        '</' identifier (':' identifier)* '>' => {
            callback_simple("on_element_end");
        };

        # Treat everything else, except for "<", as regular text. The "<" sign
        # is used for tags so we can't emit text nodes for these characters.
        ^'<'+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';

    action start_comment {
        mark = te;

        fnext comment_body;
    }

    comment_body := |*
        comment_end => {
            callback("on_comment", data, encoding, mark, te - 3);

            mark = 0;

            fnext main;
        };

        any;
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    action start_cdata {
        mark = te;

        fnext cdata_body;
    }

    cdata_body := |*
        cdata_end => {
            callback("on_cdata", data, encoding, mark, te - 3);

            mark = 0;

            fnext main;
        };

        any;
    *|;

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote+ dquote);
    string_squote = (squote ^squote+ squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #
    action start_element {
        callback_simple("on_element_start");
        fnext element_name;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment_start  => start_comment;
        cdata_start    => start_cdata;
        proc_ins_start => start_proc_ins;

        # The start of an element.
        '<' => start_element;

        # Regular closing tags.
        '</' identifier (':' identifier)* '>' => {
            callback_simple("on_element_end");
        };

        # Treat everything else, except for "<", as regular text. The "<" sign
        # is used for tags so we can't emit text nodes for these characters.
        ^'<'+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%