SSA: Fix alignment without a text position.

git-svn-id: d8f21eb7283bfb39ebab2f8f9b4a259b233f9348@546 621663c8-3916-0410-8f58-edc14a8543d5

/*
 *  SSARenderCodec.m
 *  Copyright (c) 2007 Perian Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation;
 *  version 2.1 of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */

#import "SSATagParsing.h"
#import "Categories.h"

@implementation SSAStyleSpan
-(void)dealloc
{
	ATSUDisposeStyle(astyle);
	[super dealloc];
}
@end

@implementation SSARenderEntity
-(void)dealloc
{
	int i;
	for (i=0; i < style_count; i++) [styles[i] release];
	if (disposelayout) ATSUDisposeTextLayout(layout);
	free(text);
	free(styles);
	[nstext release];
	[super dealloc];
}

-(void)increasestyles
{
	style_count++;
	styles = realloc(styles, sizeof(SSAStyleSpan*[style_count]));
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"\"%@\", %d styles, %@", nstext, style_count,
		   (posx != 1) ? [NSString stringWithFormat:@"pos (%d,%d)",posx,posy] : [NSString stringWithFormat:@"h %d v %d", halign, valign]];
}
@end

%% machine SSAtag;
%% write data;

void SetATSUStyleFlag(ATSUStyle style, ATSUAttributeTag t, Boolean v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {sizeof(v)};
	ATSUAttributeValuePtr vals[] = {&v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSUStyleOther(ATSUStyle style, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSULayoutOther(ATSUTextLayout l, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetLayoutControls(l,1,tags,sizes,vals);
}

static ATSURGBAlphaColor ParseColorTag(unsigned long c, float a)
{
	unsigned char r,g,b;
	r = c & 0xff;
	g = (c >> 8) & 0xff;
	b = (c >> 16) & 0xff;

	return (ATSURGBAlphaColor){r/255.,g/255.,b/255.,a};
}

//not worth it
/*
static void PruneIdenticalStyleSpans(SSARenderEntity *re)
{
	SSAStyleSpan *styles_new[re->style_count];

	if (re->multipart_drawing || re->style_count <= 1) return;

	size_t style_count_new = 1, remaining = re->style_count-1;
	int i=1;

	styles_new[0] = re->styles[0];

	while (remaining) {
		ATSUStyleComparison asc;
		ATSUCompareStyles(styles_new[style_count_new-1]->astyle,re->styles[i]->astyle,&asc);

		if (asc == kATSUStyleEquals) {
            styles_new[style_count_new-1]->range.length += re->styles[i]->range.length;
			[re->styles[i] release];
            re->styles[i] = nil;
		} else {
			styles_new[style_count_new++] = re->styles[i];
		}

		i++;
		remaining--;
	}

	if (style_count_new != re->style_count) {
		re->styles = realloc(re->styles, sizeof(SSAStyleSpan*) * style_count_new);
		memcpy(re->styles, styles_new, sizeof(SSAStyleSpan*) * style_count_new);

		re->style_count = style_count_new;
	}
}
*/

static void UpdateAlignment(int inum, int cur_posx, int *valign, int *halign, ATSUTextLayout cur_layout)
{
	int cur_halign, cur_valign;

	switch (inum)
	{case 1: case 4: case 7: cur_halign = S_LeftAlign; break;
	case 2: case 5: case 8: default: cur_halign = S_CenterAlign; break;
	case 3: case 6: case 9: cur_halign = S_RightAlign;}

	switch (inum)
	{case 1: case 2: case 3: default: cur_valign = S_BottomAlign; break;
	case 4: case 5: case 6: cur_valign = S_MiddleAlign; break;
	case 7: case 8: case 9: cur_valign = S_TopAlign;}

	*halign = cur_halign;
	*valign = cur_valign;
	Fract alignment;

	switch(cur_halign) {
		case S_LeftAlign:
			alignment = FloatToFract(0.);
			break;
		case S_CenterAlign: default:
			alignment = kATSUCenterAlignment;
			break;
		case S_RightAlign:
			alignment = FloatToFract(1.);
	}
	SetATSULayoutOther(cur_layout, kATSULineFlushFactorTag, sizeof(Fract), &alignment);
}

NSArray *ParseSubPacket(NSString *str, SSADocument *ssa, Boolean plaintext)
{
	NSArray *linea;
	int i=0, j, pcount = 1; unsigned len;
	NSMutableArray *rentities = [NSMutableArray array];

	if (plaintext) {
		linea = [NSArray arrayWithObject:str];
	} else {
		linea =  [str componentsSeparatedByString:@"\n"];
		pcount = [linea count];
	}

	for (; i < pcount; i ++) {
		SSARenderEntity *re = [[[SSARenderEntity alloc] init] autorelease];
		BOOL ldirty = NO;

		if (plaintext) {
			re->style = ssa->defaultstyle;
			re->layout = re->style->layout;
			re->layer = 0;
			re->marginl = re->style->marginl;
			re->marginr = re->style->marginl;
			re->marginv = re->style->marginl;
			re->usablewidth = re->style->usablewidth;
			re->halign = 1;
			re->valign = 0;
			re->nstext = [linea objectAtIndex:i];
			re->styles = NULL;
			re->disposelayout = NO;
		} else {
			NSArray *ar = [[linea objectAtIndex:(ssa->collisiontype == NormalCollisions) ? i : (pcount - i - 1)]
						   componentsSeparatedByString:@"," count:9];

			if ([ar count] < 9) continue;

			re->layer = [[ar objectAtIndex:1] intValue];
			NSString *sn = [ar objectAtIndex:2];

			re->style = ssa->defaultstyle;

			for (j=0; j < ssa->stylecount; j++) {
				if ([sn isEqualToString:ssa->styles[j].name])  {
					re->style = &ssa->styles[j];
					break;
				}
			}

			re->marginl = [[ar objectAtIndex:5] intValue];
			re->marginr = [[ar objectAtIndex:6] intValue];
			re->marginv = [[ar objectAtIndex:7] intValue];

			if (re->marginl == 0) re->marginl = re->style->marginl; else ldirty = TRUE;
			if (re->marginr == 0) re->marginr = re->style->marginr; else ldirty = TRUE;
			if (re->marginv == 0) re->marginv = re->style->marginv; else ldirty = TRUE;

			if (ldirty) {
				ATSUTextMeasurement width;
				ATSUAttributeTag tag[] = {kATSULineWidthTag};
				ByteCount		 size[] = {sizeof(ATSUTextMeasurement)};
				ATSUAttributeValuePtr val[] = {&width};

				re->usablewidth = ssa->resX - re->marginl - re->marginr;
				ATSUCreateAndCopyTextLayout(re->style->layout,&re->layout);
				width = IntToFixed(re->usablewidth);

				ATSUSetLayoutControls(re->layout,1,tag,size,val);
				re->disposelayout = YES;
			} else {
				re->usablewidth = re->style->usablewidth;
				re->layout = re->style->layout;
				re->disposelayout = NO;
			}

			re->nstext = [ar objectAtIndex:8];
		}
		len = [re->nstext length];
		re->text = malloc(sizeof(unichar[len]));

		[re->nstext getCharacters:re->text];
		re->style_count = 0;
		re->posx = re->posy = -1;
		re->halign = re->style->halign;
		re->valign = re->style->valign;
		re->multipart_drawing=NO;
		re->is_shape=NO;

#define end_re \
		cur_range = (NSRange){strbegin - pb, p - strbegin};\
		cur_range.location -= outputoffset;\
		cur_range.length -= lengthreduce;\
		if (cur_range.length) {[re increasestyles];\
		re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];\
		re->styles[re->style_count-1]->outline = cur_outline;\
		re->styles[re->style_count-1]->shadow = cur_shadow;\
		re->styles[re->style_count-1]->angle = cur_frz;\
		re->styles[re->style_count-1]->astyle = cur_style;\
		re->styles[re->style_count-1]->range = cur_range;\
		re->styles[re->style_count-1]->outlineblur = cur_be;\
		re->styles[re->style_count-1]->color = cur_color;}\
		parsetmp = [NSString stringWithCharacters:skipbegin length:p-skipbegin];\
		[output appendString:parsetmp]; \
		re->nstext = dtmp = output;\
		re->text = malloc(sizeof(unichar[[re->nstext length]]));\
		[re->nstext getCharacters:re->text];\
		[re->nstext retain];

		{
			NSRange cur_range = {0,0};
			ATSUStyle cur_style;
			ATSUTextLayout cur_layout;
			NSMutableString *output = [NSMutableString string], *dtmp;
			unichar *p = re->text, *pe = &re->text[len], *numbegin = p, *strbegin = p, *skipbegin = p, *intbegin = p, *pb = p, *posbegin=p, *strparambegin=p;
			float num, cur_outline = re->style->outline, cur_shadow = re->style->shadow, cur_scalex = re->style->scalex, cur_scaley = re->style->scaley, cur_frz=re->style->angle;
			NSString *parsetmp;
			int cs, cur_valign=re->valign, cur_halign=re->halign, cur_posx=-1, cur_posy=-1;
			ssacolors cur_color = re->style->color;
			CGAffineTransform matrix;

			unsigned long inum=0;
			unsigned outputoffset=0, lengthreduce=0;
			BOOL flag=0, newLayout=FALSE, cur_be = FALSE;
			Fixed fixv;

			ATSUCreateAndCopyStyle(re->style->atsustyle,&cur_style);

			%%{
				alphtype unsigned short;

				action bold {SetATSUStyleFlag(cur_style, kATSUQDBoldfaceTag, flag);}

				action italic {SetATSUStyleFlag(cur_style, kATSUQDItalicTag, flag);}

				action uline {SetATSUStyleFlag(cur_style, kATSUQDUnderlineTag, flag);}

				action strike {SetATSUStyleFlag(cur_style, kATSUStyleStrikeThroughTag, flag);}

				action bordersize {cur_outline = num; re->multipart_drawing = YES;}

				action shadowsize {cur_shadow = num; re->multipart_drawing = YES;}

				action bluredge {cur_be = flag;}

				action frot {cur_frz = num; re->multipart_drawing = YES;}

				action fsize {
					num *= (72./96.); // scale from Windows 96dpi
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUSizeTag, sizeof(fixv), &fixv);
				}

				action ftrack {
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUTrackingTag, sizeof(fixv), &fixv);
				}

				action fscalex {
					cur_scalex = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action fscaley {
					cur_scaley = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action strp_begin {
					strparambegin = p;
				}

				action fontname {
					ATSUFontID	font;
					font = FMGetFontFromATSFontRef(ATSFontFindFromName((CFStringRef)[NSString stringWithCharacters:strparambegin length:p-strparambegin],kATSOptionFlagsDefault));
					SetATSUStyleOther(cur_style, kATSUFontTag, sizeof(ATSUFontID), &font);
				}

				action alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(inum, cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action ssa_alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(SSA2ASSAlignment(inum), cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action pos_begin {
					posbegin=p;
				}

				action pos_end {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}
					NSArray *coo = [[NSString stringWithCharacters:posbegin length:p-posbegin] componentsSeparatedByString:@","];
					cur_posx = [[coo objectAtIndex:0] intValue];
					cur_posy = [[coo objectAtIndex:1] intValue];
				}

				action primarycolor {
					cur_color.primary = ParseColorTag(inum,cur_color.primary.alpha);
					re->multipart_drawing = YES;
				}

				action secondarycolor {
					cur_color.secondary = ParseColorTag(inum,cur_color.secondary.alpha);
					re->multipart_drawing = YES;
				}

				action outlinecolor {
					cur_color.outline = ParseColorTag(inum,cur_color.outline.alpha);
					re->multipart_drawing = YES;
				}

				action shadowcolor {
					cur_color.shadow = ParseColorTag(inum,cur_color.shadow.alpha);
					re->multipart_drawing = YES;
				}

				action primaryalpha {
					cur_color.primary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action secondaryalpha {
					cur_color.secondary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action outlinealpha {
					cur_color.outline.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action shadowalpha {
					cur_color.shadow.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action nl_handler {
					NSString *append = @"\n";
					parsetmp = [NSString stringWithCharacters:skipbegin length:(p-2)-skipbegin];
					[output appendString:parsetmp];

					skipbegin = p;

					if (*(p - 1) == 'h') append = @" ";

					[output appendString:append];

					lengthreduce++;
				}

				action enter_tag {
					unsigned taglen = p - skipbegin;
					parsetmp = [NSString stringWithCharacters:skipbegin length:taglen];
					[output appendString:parsetmp];
					skipbegin = p;

					cur_range = (NSRange){strbegin - pb, p - strbegin};
					cur_range.location -= outputoffset;
					cur_range.length -= lengthreduce;

					outputoffset += lengthreduce;
					lengthreduce = 0;

                    if (cur_range.length) {
                        [re increasestyles];
                        re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];
                        re->styles[re->style_count-1]->outline = cur_outline;
                        re->styles[re->style_count-1]->shadow = cur_shadow;
                        re->styles[re->style_count-1]->angle = cur_frz;
                        re->styles[re->style_count-1]->astyle = cur_style;
                        re->styles[re->style_count-1]->range = cur_range;
                        re->styles[re->style_count-1]->outlineblur = cur_be;
                        re->styles[re->style_count-1]->color = cur_color;
                        ATSUCreateAndCopyStyle(cur_style,&cur_style);
                    }
				}

				action exit_tag {
					// XXX exit_tag ideally should run one char later, so we just pretend it does
					p++;
					outputoffset += p - skipbegin;
					skipbegin = p;
					strbegin = p;

					if (newLayout) {
						newLayout = FALSE;
						end_re;

						if ([re->nstext length] != 0) [rentities addObject:re];

						SSARenderEntity *nre = [[[SSARenderEntity alloc] init] autorelease];
						nre->layer = re->layer;
						nre->style = re->style;
						nre->marginl = re->marginl; nre->marginr = re->marginr; nre->marginv = re->marginv; nre->usablewidth = re->usablewidth;
						nre->posx = cur_posx; nre->posy = cur_posy; nre->halign = cur_halign; nre->valign = cur_valign;
						nre->nstext = nil;
						nre->text = nil;
						nre->layout = cur_layout;
						nre->styles = malloc(0);
						nre->style_count = 0;
						nre->disposelayout = YES;
						nre->multipart_drawing = NO;
						nre->is_shape = re->is_shape;
						re = nre;
					}
					p--;
				}

				action color_end {
					NSString *hexn = [NSString stringWithCharacters:intbegin length:p-intbegin];
					inum = strtoul([hexn UTF8String], NULL, 16);
				}

				action styleset {
					re->multipart_drawing = YES;
					ssastyleline *the_style = re->style;
					NSString *searchsn = [NSString stringWithCharacters:strparambegin length:p-strparambegin];

					if ([searchsn length] > 0) {
						for (j=0; j < ssa->stylecount; j++) {
							if ([searchsn isEqualToString:ssa->styles[j].name])  {
								the_style = &ssa->styles[j];
								break;
							}
						}
					}

					ATSUCopyAttributes(the_style->atsustyle,cur_style);
					cur_color = the_style->color;
					cur_outline = the_style->outline;
					cur_shadow = the_style->shadow;
					cur_frz = the_style->angle;
					cur_scalex = cur_scaley = 1;
				}

				action skip_t_tag {
					while (p != pe && *p != ')' && *p != '}') p++;
					p++;
				}

				action draw_mode {
					re->is_shape = inum != 0;
				}

				flag = ([01] % {unichar fl = *(p-1); if (fl == '0' || fl == '1') flag = fl - '0';})? > {flag = 0;};
				num_ = "-"? digit+ ('.' digit*)?;
				num = num_ > {numbegin = p;} % {num = [[NSString stringWithCharacters:numbegin length:p-numbegin] doubleValue];};

				intnum = "-"? (digit+) > {intbegin = p;} % {inum = [[NSString stringWithCharacters:intbegin length:p-intbegin] intValue];};

				color = ("H"|"&"){,2} (xdigit+) > {intbegin = p;} % color_end "&";

				cmd_specific = (("pos(" [^)]+ > pos_begin ")" % pos_end)
								|"bord" num %bordersize
								|"b" flag %bold
								|"be" flag %bluredge
								|"i" flag %italic
								|"u" flag %uline
								|"s" flag %strike
								|"fs" num %fsize
								|"fsp" num %ftrack
								|"fscx" num %fscalex
								|"fscy" num %fscaley
								|("fr" "z"? num %frot)
								|("fn" [^\\}]* > strp_begin %fontname)
								|"shad" num %shadowsize
								|"a" intnum %ssa_alignment
								|"an" intnum %alignment
								|"c" color %primarycolor
								|"1c" color %primarycolor
								|"2c" color %secondarycolor
								|"3c" color %outlinecolor
								|"4c" color %shadowcolor
								|("1a"|"alpha") color %primaryalpha
								|"2a" color %secondaryalpha
								|"3a" color %outlinealpha
								|"4a" color %shadowalpha
								|("r" [^\\}]* > strp_begin %styleset)
								|("fe"|"k"|"kf"|"K"|"ko"|"q"|"fr"|"fad"|"move"|"clip"|"o"|"frx"|"fry") [^\\}]*
								|"p" num %draw_mode
								#|"t" [^)}]* # enabling this crashes ragel
								|"t(" % skip_t_tag
								|""
								);

				cmd = ("\\" :> cmd_specific)+;

				plaintext = [^}]*;

				tag = "{" (cmd | plaintext) % exit_tag "}";

				nl = "\\" [Nnh];

				special = nl % nl_handler | (tag > enter_tag);

				text = any*;
				main := (text special?)*;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (pb[len-1] == '}') skipbegin = p; // make up for how exit_tag isn't called if the } is the last char in the line

			end_re;
			if (!cur_range.length) ATSUDisposeStyle(cur_style);
            //PruneIdenticalStyleSpans(re);

			free(pb);
			if ([re->nstext length] != 0) [rentities addObject:re];
		}

	}
	return rentities;
}

/*
 *  SSARenderCodec.m
 *  Copyright (c) 2007 Perian Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation;
 *  version 2.1 of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */

#import "SSATagParsing.h"
#import "Categories.h"

@implementation SSAStyleSpan
-(void)dealloc
{
	ATSUDisposeStyle(astyle);
	[super dealloc];
}
@end

@implementation SSARenderEntity
-(void)dealloc
{
	int i;
	for (i=0; i < style_count; i++) [styles[i] release];
	if (disposelayout) ATSUDisposeTextLayout(layout);
	free(text);
	free(styles);
	[nstext release];
	[super dealloc];
}

-(void)increasestyles
{
	style_count++;
	styles = realloc(styles, sizeof(SSAStyleSpan*[style_count]));
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"\"%@\", %d styles, %@", nstext, style_count,
		   (posx != 1) ? [NSString stringWithFormat:@"pos (%d,%d)",posx,posy] : [NSString stringWithFormat:@"h %d v %d", halign, valign]];
}
@end

%% machine SSAtag;
%% write data;

void SetATSUStyleFlag(ATSUStyle style, ATSUAttributeTag t, Boolean v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {sizeof(v)};
	ATSUAttributeValuePtr vals[] = {&v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSUStyleOther(ATSUStyle style, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSULayoutOther(ATSUTextLayout l, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetLayoutControls(l,1,tags,sizes,vals);
}

static ATSURGBAlphaColor ParseColorTag(unsigned long c, float a)
{
	unsigned char r,g,b;
	r = c & 0xff;
	g = (c >> 8) & 0xff;
	b = (c >> 16) & 0xff;

	return (ATSURGBAlphaColor){r/255.,g/255.,b/255.,a};
}

//not worth it
/*
static void PruneIdenticalStyleSpans(SSARenderEntity *re)
{
	SSAStyleSpan *styles_new[re->style_count];

	if (re->multipart_drawing || re->style_count <= 1) return;

	size_t style_count_new = 1, remaining = re->style_count-1;
	int i=1;

	styles_new[0] = re->styles[0];

	while (remaining) {
		ATSUStyleComparison asc;
		ATSUCompareStyles(styles_new[style_count_new-1]->astyle,re->styles[i]->astyle,&asc);

		if (asc == kATSUStyleEquals) {
            styles_new[style_count_new-1]->range.length += re->styles[i]->range.length;
			[re->styles[i] release];
            re->styles[i] = nil;
		} else {
			styles_new[style_count_new++] = re->styles[i];
		}

		i++;
		remaining--;
	}

	if (style_count_new != re->style_count) {
		re->styles = realloc(re->styles, sizeof(SSAStyleSpan*) * style_count_new);
		memcpy(re->styles, styles_new, sizeof(SSAStyleSpan*) * style_count_new);

		re->style_count = style_count_new;
	}
}
*/

static void UpdateAlignment(int inum, int cur_posx, int *valign, int *halign, ATSUTextLayout cur_layout)
{
	int cur_halign, cur_valign;

	switch (inum)
	{case 1: case 4: case 7: cur_halign = S_LeftAlign; break;
	case 2: case 5: case 8: default: cur_halign = S_CenterAlign; break;
	case 3: case 6: case 9: cur_halign = S_RightAlign;}

	switch (inum)
	{case 1: case 2: case 3: default: cur_valign = S_BottomAlign; break;
	case 4: case 5: case 6: cur_valign = S_MiddleAlign; break;
	case 7: case 8: case 9: cur_valign = S_TopAlign;}

	*halign = cur_halign;
	*valign = cur_valign;
	Fract alignment;

	if (cur_posx != -1) {
		switch(cur_halign) {
			case S_LeftAlign:
				alignment = FloatToFract(0.);
				break;
			case S_CenterAlign: default:
				alignment = kATSUCenterAlignment;
				break;
			case S_RightAlign:
				alignment = FloatToFract(1.);
		}
		SetATSULayoutOther(cur_layout, kATSULineFlushFactorTag, sizeof(Fract), &alignment);
	}
}

NSArray *ParseSubPacket(NSString *str, SSADocument *ssa, Boolean plaintext)
{
	NSArray *linea;
	int i=0, j, pcount = 1; unsigned len;
	NSMutableArray *rentities = [NSMutableArray array];

	if (plaintext) {
		linea = [NSArray arrayWithObject:str];
	} else {
		linea =  [str componentsSeparatedByString:@"\n"];
		pcount = [linea count];
	}

	for (; i < pcount; i ++) {
		SSARenderEntity *re = [[[SSARenderEntity alloc] init] autorelease];
		BOOL ldirty = NO;

		if (plaintext) {
			re->style = ssa->defaultstyle;
			re->layout = re->style->layout;
			re->layer = 0;
			re->marginl = re->style->marginl;
			re->marginr = re->style->marginl;
			re->marginv = re->style->marginl;
			re->usablewidth = re->style->usablewidth;
			re->halign = 1;
			re->valign = 0;
			re->nstext = [linea objectAtIndex:i];
			re->styles = NULL;
			re->disposelayout = NO;
		} else {
			NSArray *ar = [[linea objectAtIndex:(ssa->collisiontype == NormalCollisions) ? i : (pcount - i - 1)]
						   componentsSeparatedByString:@"," count:9];

			if ([ar count] < 9) continue;

			re->layer = [[ar objectAtIndex:1] intValue];
			NSString *sn = [ar objectAtIndex:2];

			re->style = ssa->defaultstyle;

			for (j=0; j < ssa->stylecount; j++) {
				if ([sn isEqualToString:ssa->styles[j].name])  {
					re->style = &ssa->styles[j];
					break;
				}
			}

			re->marginl = [[ar objectAtIndex:5] intValue];
			re->marginr = [[ar objectAtIndex:6] intValue];
			re->marginv = [[ar objectAtIndex:7] intValue];

			if (re->marginl == 0) re->marginl = re->style->marginl; else ldirty = TRUE;
			if (re->marginr == 0) re->marginr = re->style->marginr; else ldirty = TRUE;
			if (re->marginv == 0) re->marginv = re->style->marginv; else ldirty = TRUE;

			if (ldirty) {
				ATSUTextMeasurement width;
				ATSUAttributeTag tag[] = {kATSULineWidthTag};
				ByteCount		 size[] = {sizeof(ATSUTextMeasurement)};
				ATSUAttributeValuePtr val[] = {&width};

				re->usablewidth = ssa->resX - re->marginl - re->marginr;
				ATSUCreateAndCopyTextLayout(re->style->layout,&re->layout);
				width = IntToFixed(re->usablewidth);

				ATSUSetLayoutControls(re->layout,1,tag,size,val);
				re->disposelayout = YES;
			} else {
				re->usablewidth = re->style->usablewidth;
				re->layout = re->style->layout;
				re->disposelayout = NO;
			}

			re->nstext = [ar objectAtIndex:8];
		}
		len = [re->nstext length];
		re->text = malloc(sizeof(unichar[len]));

		[re->nstext getCharacters:re->text];
		re->style_count = 0;
		re->posx = re->posy = -1;
		re->halign = re->style->halign;
		re->valign = re->style->valign;
		re->multipart_drawing=NO;
		re->is_shape=NO;

#define end_re \
		cur_range = (NSRange){strbegin - pb, p - strbegin};\
		cur_range.location -= outputoffset;\
		cur_range.length -= lengthreduce;\
		if (cur_range.length) {[re increasestyles];\
		re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];\
		re->styles[re->style_count-1]->outline = cur_outline;\
		re->styles[re->style_count-1]->shadow = cur_shadow;\
		re->styles[re->style_count-1]->angle = cur_frz;\
		re->styles[re->style_count-1]->astyle = cur_style;\
		re->styles[re->style_count-1]->range = cur_range;\
		re->styles[re->style_count-1]->outlineblur = cur_be;\
		re->styles[re->style_count-1]->color = cur_color;}\
		parsetmp = [NSString stringWithCharacters:skipbegin length:p-skipbegin];\
		[output appendString:parsetmp]; \
		re->nstext = dtmp = output;\
		re->text = malloc(sizeof(unichar[[re->nstext length]]));\
		[re->nstext getCharacters:re->text];\
		[re->nstext retain];

		{
			NSRange cur_range = {0,0};
			ATSUStyle cur_style;
			ATSUTextLayout cur_layout;
			NSMutableString *output = [NSMutableString string], *dtmp;
			unichar *p = re->text, *pe = &re->text[len], *numbegin = p, *strbegin = p, *skipbegin = p, *intbegin = p, *pb = p, *posbegin=p, *strparambegin=p;
			float num, cur_outline = re->style->outline, cur_shadow = re->style->shadow, cur_scalex = re->style->scalex, cur_scaley = re->style->scaley, cur_frz=re->style->angle;
			NSString *parsetmp;
			int cs, cur_valign=re->valign, cur_halign=re->halign, cur_posx=-1, cur_posy=-1;
			ssacolors cur_color = re->style->color;
			CGAffineTransform matrix;

			unsigned long inum=0;
			unsigned outputoffset=0, lengthreduce=0;
			BOOL flag=0, newLayout=FALSE, cur_be = FALSE;
			Fixed fixv;

			ATSUCreateAndCopyStyle(re->style->atsustyle,&cur_style);

			%%{
				alphtype unsigned short;

				action bold {SetATSUStyleFlag(cur_style, kATSUQDBoldfaceTag, flag);}

				action italic {SetATSUStyleFlag(cur_style, kATSUQDItalicTag, flag);}

				action uline {SetATSUStyleFlag(cur_style, kATSUQDUnderlineTag, flag);}

				action strike {SetATSUStyleFlag(cur_style, kATSUStyleStrikeThroughTag, flag);}

				action bordersize {cur_outline = num; re->multipart_drawing = YES;}

				action shadowsize {cur_shadow = num; re->multipart_drawing = YES;}

				action bluredge {cur_be = flag;}

				action frot {cur_frz = num; re->multipart_drawing = YES;}

				action fsize {
					num *= (72./96.); // scale from Windows 96dpi
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUSizeTag, sizeof(fixv), &fixv);
				}

				action ftrack {
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUTrackingTag, sizeof(fixv), &fixv);
				}

				action fscalex {
					cur_scalex = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action fscaley {
					cur_scaley = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action strp_begin {
					strparambegin = p;
				}

				action fontname {
					ATSUFontID	font;
					font = FMGetFontFromATSFontRef(ATSFontFindFromName((CFStringRef)[NSString stringWithCharacters:strparambegin length:p-strparambegin],kATSOptionFlagsDefault));
					SetATSUStyleOther(cur_style, kATSUFontTag, sizeof(ATSUFontID), &font);
				}

				action alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(inum, cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action ssa_alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(SSA2ASSAlignment(inum), cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action pos_begin {
					posbegin=p;
				}

				action pos_end {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}
					NSArray *coo = [[NSString stringWithCharacters:posbegin length:p-posbegin] componentsSeparatedByString:@","];
					cur_posx = [[coo objectAtIndex:0] intValue];
					cur_posy = [[coo objectAtIndex:1] intValue];
				}

				action primarycolor {
					cur_color.primary = ParseColorTag(inum,cur_color.primary.alpha);
					re->multipart_drawing = YES;
				}

				action secondarycolor {
					cur_color.secondary = ParseColorTag(inum,cur_color.secondary.alpha);
					re->multipart_drawing = YES;
				}

				action outlinecolor {
					cur_color.outline = ParseColorTag(inum,cur_color.outline.alpha);
					re->multipart_drawing = YES;
				}

				action shadowcolor {
					cur_color.shadow = ParseColorTag(inum,cur_color.shadow.alpha);
					re->multipart_drawing = YES;
				}

				action primaryalpha {
					cur_color.primary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action secondaryalpha {
					cur_color.secondary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action outlinealpha {
					cur_color.outline.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action shadowalpha {
					cur_color.shadow.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action nl_handler {
					NSString *append = @"\n";
					parsetmp = [NSString stringWithCharacters:skipbegin length:(p-2)-skipbegin];
					[output appendString:parsetmp];

					skipbegin = p;

					if (*(p - 1) == 'h') append = @" ";

					[output appendString:append];

					lengthreduce++;
				}

				action enter_tag {
					unsigned taglen = p - skipbegin;
					parsetmp = [NSString stringWithCharacters:skipbegin length:taglen];
					[output appendString:parsetmp];
					skipbegin = p;

					cur_range = (NSRange){strbegin - pb, p - strbegin};
					cur_range.location -= outputoffset;
					cur_range.length -= lengthreduce;

					outputoffset += lengthreduce;
					lengthreduce = 0;

                    if (cur_range.length) {
                        [re increasestyles];
                        re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];
                        re->styles[re->style_count-1]->outline = cur_outline;
                        re->styles[re->style_count-1]->shadow = cur_shadow;
                        re->styles[re->style_count-1]->angle = cur_frz;
                        re->styles[re->style_count-1]->astyle = cur_style;
                        re->styles[re->style_count-1]->range = cur_range;
                        re->styles[re->style_count-1]->outlineblur = cur_be;
                        re->styles[re->style_count-1]->color = cur_color;
                        ATSUCreateAndCopyStyle(cur_style,&cur_style);
                    }
				}

				action exit_tag {
					// XXX exit_tag ideally should run one char later, so we just pretend it does
					p++;
					outputoffset += p - skipbegin;
					skipbegin = p;
					strbegin = p;

					if (newLayout) {
						newLayout = FALSE;
						end_re;

						if ([re->nstext length] != 0) [rentities addObject:re];

						SSARenderEntity *nre = [[[SSARenderEntity alloc] init] autorelease];
						nre->layer = re->layer;
						nre->style = re->style;
						nre->marginl = re->marginl; nre->marginr = re->marginr; nre->marginv = re->marginv; nre->usablewidth = re->usablewidth;
						nre->posx = cur_posx; nre->posy = cur_posy; nre->halign = cur_halign; nre->valign = cur_valign;
						nre->nstext = nil;
						nre->text = nil;
						nre->layout = cur_layout;
						nre->styles = malloc(0);
						nre->style_count = 0;
						nre->disposelayout = YES;
						nre->multipart_drawing = NO;
						nre->is_shape = re->is_shape;
						re = nre;
					}
					p--;
				}

				action color_end {
					NSString *hexn = [NSString stringWithCharacters:intbegin length:p-intbegin];
					inum = strtoul([hexn UTF8String], NULL, 16);
				}

				action styleset {
					re->multipart_drawing = YES;
					ssastyleline *the_style = re->style;
					NSString *searchsn = [NSString stringWithCharacters:strparambegin length:p-strparambegin];

					if ([searchsn length] > 0) {
						for (j=0; j < ssa->stylecount; j++) {
							if ([searchsn isEqualToString:ssa->styles[j].name])  {
								the_style = &ssa->styles[j];
								break;
							}
						}
					}

					ATSUCopyAttributes(the_style->atsustyle,cur_style);
					cur_color = the_style->color;
					cur_outline = the_style->outline;
					cur_shadow = the_style->shadow;
					cur_frz = the_style->angle;
					cur_scalex = cur_scaley = 1;
				}

				action skip_t_tag {
					while (p != pe && *p != ')' && *p != '}') p++;
					p++;
				}

				action draw_mode {
					re->is_shape = inum != 0;
				}

				flag = ([01] % {unichar fl = *(p-1); if (fl == '0' || fl == '1') flag = fl - '0';})? > {flag = 0;};
				num_ = "-"? digit+ ('.' digit*)?;
				num = num_ > {numbegin = p;} % {num = [[NSString stringWithCharacters:numbegin length:p-numbegin] doubleValue];};

				intnum = "-"? (digit+) > {intbegin = p;} % {inum = [[NSString stringWithCharacters:intbegin length:p-intbegin] intValue];};

				color = ("H"|"&"){,2} (xdigit+) > {intbegin = p;} % color_end "&";

				cmd_specific = (("pos(" [^)]+ > pos_begin ")" % pos_end)
								|"bord" num %bordersize
								|"b" flag %bold
								|"be" flag %bluredge
								|"i" flag %italic
								|"u" flag %uline
								|"s" flag %strike
								|"fs" num %fsize
								|"fsp" num %ftrack
								|"fscx" num %fscalex
								|"fscy" num %fscaley
								|("fr" "z"? num %frot)
								|("fn" [^\\}]* > strp_begin %fontname)
								|"shad" num %shadowsize
								|"a" intnum %ssa_alignment
								|"an" intnum %alignment
								|"c" color %primarycolor
								|"1c" color %primarycolor
								|"2c" color %secondarycolor
								|"3c" color %outlinecolor
								|"4c" color %shadowcolor
								|("1a"|"alpha") color %primaryalpha
								|"2a" color %secondaryalpha
								|"3a" color %outlinealpha
								|"4a" color %shadowalpha
								|("r" [^\\}]* > strp_begin %styleset)
								|("fe"|"k"|"kf"|"K"|"ko"|"q"|"fr"|"fad"|"move"|"clip"|"o"|"frx"|"fry") [^\\}]*
								|"p" num %draw_mode
								#|"t" [^)}]* # enabling this crashes ragel
								|"t(" % skip_t_tag
								|""
								);

				cmd = ("\\" :> cmd_specific)+;

				plaintext = [^}]*;

				tag = "{" (cmd | plaintext) % exit_tag "}";

				nl = "\\" [Nnh];

				special = nl % nl_handler | (tag > enter_tag);

				text = any*;
				main := (text special?)*;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (pb[len-1] == '}') skipbegin = p; // make up for how exit_tag isn't called if the } is the last char in the line

			end_re;
			if (!cur_range.length) ATSUDisposeStyle(cur_style);
            //PruneIdenticalStyleSpans(re);

			free(pb);
			if ([re->nstext length] != 0) [rentities addObject:re];
		}

	}
	return rentities;
}


Clean up after consuming buffer in stomp parser

%%{
  machine Message;

  variable data @_data;
  variable p    @_p;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  # Actions.
  action Mark { mark }

  # Common constants.
  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;

  # Message components.
  command = "CONNECT";

  message := (command > Mark @ { @message.command = consume_utf8 }) EOL EOL NULL;
}%%

module Stompede
  module Stomp
    %% write data noprefix;

    class << self
      # @param [String] data
      # @return [Stomp::Message]
      def parse(data)
        @message = Stomp::Message.new

        @_data = data.force_encoding("BINARY")
        @_p = 0
        pe = data.length
        cs = start

        %% write exec;

        @message if cs >= first_final
      end

      def mark
        @mark = @_p
      end

      def consume_utf8
        string = @_data[@mark..@_p].force_encoding("UTF-8")
        @mark = nil # signal the marked data is consumed
        string
      end
    end
  end
end

%%{
  machine Message;

  variable data @_data;
  variable p    @_p;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  # Actions.
  action Mark { mark }

  # Common constants.
  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;

  # Message components.
  command = "CONNECT";

  message := (command > Mark @ { @message.command = consume }) EOL EOL NULL;
}%%

module Stompede
  module Stomp
    %% write data noprefix;

    class << self
      # @param [String] data
      # @return [Stomp::Message]
      def parse(data)
        @message = Stomp::Message.new

        @_data = data.force_encoding("BINARY")
        @_p = 0
        pe = data.length
        cs = start

        %% write exec;

        @message if cs >= first_final
      end

      def mark
        @mark = @_p
      end

      def consume
        @_data[@mark..@_p].force_encoding("UTF-8")
      end
    end
  end
end


improve list definitions

improve list definitions

This is currently problematic since there are yields inside definitions
where there shouldn't be. This is causing list specs to succeed and set
specs to fail.

# begin: ragel
=begin
%%{
  machine bel;

  include 'identifier.rl';
  include 'string.rl';

  action list_start {
    #$stderr.puts 'list_start'
    @opened = true
    @buffers[:list] = list()
  }

  action list_finish {
    #$stderr.puts 'list_finish'
    @closed = true
  }

  action list_end {
    $stderr.puts "list_end"
    completed = @opened && @closed
    unless completed
      @buffers[:list] = list(complete: false)
    else
      @buffers[:list].complete = completed
    end
  }

  action list_yield {
    #$stderr.puts "list_yield"
    yield @buffers[:list]
  }

  action clear {
    #$stderr.puts "clear"
    @buffers.delete(:string)
    @buffers.delete(:ident)
  }

  action string {
    #$stderr.puts "string"
    @buffers[:list_arg] = list_item(@buffers[:string])
  }

  action ident {
    #$stderr.puts "ident"
    @buffers[:list_arg] = list_item(@buffers[:ident])
  }

  action start_list {
    #$stderr.puts "start_list"
    @buffers[:list] = list()
  }

  action append_list {
    #$stderr.puts "append_list"
    # Append list argument if its value is not empty.
    if @buffers[:list_arg]
      list_arg_value = @buffers[:list_arg].children[0].children[0]
      if list_arg_value != ''
        @buffers[:list] <<= @buffers[:list_arg]
      end
    end
  }

  action finish_list {
    #$stderr.puts "finish_list"
    #TODO: Mark @buffers[:list] as complete.
  }

  action error_list_string {
    #$stderr.puts "error_list_string"
    #TODO: Mark @buffers[:list_arg] string as error.
    @buffers[:list_arg] = list_item(@buffers[:string])
  }

  action error_list_ident {
    #$stderr.puts "error_list_ident"
    #TODO: Mark @buffers[:list_arg] identifier as error.
    @buffers[:list_arg] = list_item(@buffers[:ident])
  }

  action yield_complete_list {
    #$stderr.puts "yield_complete_list"
    #TODO: Mark @buffers[:list_arg] identifier as error.
    yield @buffers[:list]
  }

  action yield_error_list {
    #$stderr.puts "yield_error_list"
    @buffers[:list] ||= list()
    yield @buffers[:list]
  }

  action start_list_arg {
    #$stderr.puts "start_list_arg"
    @incomplete[:list_arg] = []
  }

  action accum_list_arg {
    #$stderr.puts "accum_list_arg"
    @incomplete[:list_arg] << fc
  }

  action end_list_arg {
    # finished list arg
    @incomplete.delete(:list_arg)
    if @buffers.key?(:string)
      ast_node = @buffers.delete(:string)
    else
      ast_node = @buffers.delete(:ident)
    end
    @buffers[:list] <<= list_item(ast_node, complete: true)
  }

  action eof_list_arg {
    # unfinished list arg
    $stderr.puts 'eof_list_arg'
    arg = @incomplete.delete(:list_arg)
    if @incomplete.key?(:string)
      ast_node = string(utf8_string(arg), complete: false)
    else
      ast_node = identifier(utf8_string(arg), complete: false)
    end
    @buffers[:list] <<= list_item(ast_node, complete: false)
    @buffers[:list].complete = false
    yield @buffers[:list]
  }

  action err_list_arg {
    # unfinished list arg
    #$stderr.puts 'err_list_arg'
    arg = @incomplete.delete(:list_arg)
    if @incomplete.key?(:string)
      ast_node = string(utf8_string(arg), complete: false)
    else
      ast_node = identifier(utf8_string(arg), complete: false)
    end
    @buffers[:list] <<= list_item(ast_node, complete: false)
    @buffers[:list].complete = false
    yield @buffers[:list]
  }

  action err_members {
    # unfinished list arg
    $stderr.puts 'err_members'
    arg = @incomplete.delete(:list_arg) || []
    if @incomplete.key?(:string)
      ast_node = string(utf8_string(arg), complete: false)
    else
      ast_node = identifier(utf8_string(arg), complete: false)
    end
    @buffers[:list] <<= list_item(ast_node, complete: false)
    @buffers[:list].complete = false
    yield @buffers[:list]
  }

  action eof_members {
    $stderr.puts 'eof_members'
    # unfinished members
    #$stderr.puts "incomplete members"
    #$stderr.puts @buffers[:string]
  }

  action err_list {
    $stderr.puts "err_list"
  }

  action eof_list {
    $stderr.puts 'eof_list'
    unless @closed
      $stderr.puts "incomplete list - why?"
    else
      #$stderr.puts "complete list"
    end
  }

  start_list = '{' SP* %list_start;
  end_list = SP* '}' %list_finish;

  ident_list_member = id_ident
                      <to(accum_list_arg)
                      $err(err_list_arg)
                      ;
  string_list_member = str_string
                      <to(accum_list_arg)
                      $err(err_list_arg)
                      ;
  list_member = (string_list_member | ident_list_member)
                >start_list_arg
                %end_list_arg
                $err(err_list_arg)
                $eof(eof_list_arg)
                ;
  members = (list_member (COMMA_DELIM list_member)*)
            $eof(eof_members)
            $err(err_members)
            ;

  lst_list =
    start_list
    members
    end_list
    %list_end
    $eof(eof_list)
    $err(err_list)
    ;

  list_ast_new := (start_list members end_list |
                  start_list end_list |
                  start_list |
                  start_list?)
                  NL?
                  $list_end
                  $list_yield
                  $eof(eof_list)
                  ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @opened     = false
            @closed     = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'identifier.rl';
  include 'string.rl';

  action list_start {
    #$stderr.puts 'list_start'
    @opened = true
    @buffers[:list] = list()
  }

  action list_finish {
    #$stderr.puts 'list_finish'
    @closed = true
  }

  action list_end {
    #$stderr.puts "list_end"
    completed = @opened && @closed
    unless completed
      @buffers[:list] = list(complete: false)
    else
      @buffers[:list].complete = completed
    end
  }

  action list_yield {
    #$stderr.puts "list_yield"
    yield @buffers[:list]
  }

  action clear {
    #$stderr.puts "clear"
    @buffers.delete(:string)
    @buffers.delete(:ident)
  }

  action string {
    #$stderr.puts "string"
    @buffers[:list_arg] = list_item(@buffers[:string])
  }

  action ident {
    #$stderr.puts "ident"
    @buffers[:list_arg] = list_item(@buffers[:ident])
  }

  action start_list {
    #$stderr.puts "start_list"
    @buffers[:list] = list()
  }

  action append_list {
    #$stderr.puts "append_list"
    # Append list argument if its value is not empty.
    if @buffers[:list_arg]
      list_arg_value = @buffers[:list_arg].children[0].children[0]
      if list_arg_value != ''
        @buffers[:list] <<= @buffers[:list_arg]
      end
    end
  }

  action finish_list {
    #$stderr.puts "finish_list"
    #TODO: Mark @buffers[:list] as complete.
  }

  action error_list_string {
    #$stderr.puts "error_list_string"
    #TODO: Mark @buffers[:list_arg] string as error.
    @buffers[:list_arg] = list_item(@buffers[:string])
  }

  action error_list_ident {
    #$stderr.puts "error_list_ident"
    #TODO: Mark @buffers[:list_arg] identifier as error.
    @buffers[:list_arg] = list_item(@buffers[:ident])
  }

  action yield_complete_list {
    #$stderr.puts "yield_complete_list"
    #TODO: Mark @buffers[:list_arg] identifier as error.
    yield @buffers[:list]
  }

  action yield_error_list {
    #$stderr.puts "yield_error_list"
    @buffers[:list] ||= list()
    yield @buffers[:list]
  }

  action start_list_arg {
    #$stderr.puts "start_list_arg"
    @incomplete[:list_arg] = []
  }

  action accum_list_arg {
    #$stderr.puts "accum_list_arg"
    @incomplete[:list_arg] << fc
  }

  action end_list_arg {
    #$stderr.puts "end_list_arg"
    # finished list arg
    @incomplete.delete(:list_arg)
    if @buffers.key?(:string)
      ast_node = @buffers.delete(:string)
    else
      ast_node = @buffers.delete(:ident)
    end
    @buffers[:list] <<= list_item(ast_node, complete: true)
  }

  action eof_list_arg {
    # unfinished list arg
    #$stderr.puts 'eof_list_arg'
    arg = @incomplete.delete(:list_arg)
    if @incomplete.key?(:string)
      ast_node = string(utf8_string(arg), complete: false)
    else
      ast_node = identifier(utf8_string(arg), complete: false)
    end
    @buffers[:list] <<= list_item(ast_node, complete: false)
    @buffers[:list].complete = false
    yield @buffers[:list]
  }

  action eof_members {
    #$stderr.puts 'eof_members'
    # unfinished members
    #$stderr.puts "incomplete members"
    #$stderr.puts @buffers[:string]
  }

  action eof_list {
    #$stderr.puts 'eof_list'
    unless @closed
      $stderr.puts "incomplete list - why?"
    else
      #$stderr.puts "complete list"
    end
  }

  start_list = '{' SP* %list_start;
  end_list = SP* '}' %list_finish;

  ident_list_member = id_ident
                      <to(accum_list_arg)
                      ;
  string_list_member = str_string
                      <to(accum_list_arg)
                      ;
  list_member = (string_list_member | ident_list_member)
                >start_list_arg
                %end_list_arg
                $eof(eof_list_arg)
                ;
  members = (list_member (COMMA_DELIM list_member)*)
            $eof(eof_members)
            ;

  list = start_list end_list @list_end;

  list_ast_new := (start_list members end_list |
                  start_list end_list |
                  start_list |
                  start_list?)
                  NL?
                  $list_end
                  $list_yield
                  $eof(eof_list)
                  ;

  LIST  =
    '{' @start_list
    SP*
    (
      str_string %string $err(error_list_string) |
      id_ident  %ident  $err(error_list_ident)
    ) $err(append_list) %append_list
    SP*
    (
      ',' @clear
      SP*
      (
        str_string %string $err(error_list_string) |
        id_ident  %ident  $err(error_list_ident)
      ) $err(append_list) %append_list
      SP*
    )*
    '}' @finish_list;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @opened     = false
            @closed     = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Fix enormous ragel output

Fix enormous ragel output

#include "toml.h"
#include "toml_private.h"

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>
#include <math.h>
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

bool add_node_to_tree(struct list_head* list_stack, struct toml_node* cur_table, struct toml_node* node, char* name, char** parse_error, int* malloc_error, int cur_line)
{
	struct toml_stack_item *cur_list =
					list_tail(list_stack, struct toml_stack_item, list);

	if (cur_list) {
		if (cur_list->list_type && cur_list->list_type != node->type) {
			asprintf(parse_error,
					"incompatible types list %s this %s line %d\n",
					toml_type_to_str(cur_list->list_type),
					toml_type_to_str(node->type), cur_line);
			return false;
		}
		cur_list->list_type = node->type;

		struct toml_list_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}

		memcpy(&item->node, node, sizeof(*node));
		item->node.name = NULL;
		list_add_tail(&cur_list->node->value.list, &item->list);
	} else {
		struct toml_table_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}
		memcpy(&item->node, node, sizeof(*node));
		item->node.name = name;
		list_add_tail(&cur_table->value.map, &item->map);
	}

	return true;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - ('#'|'='|'"'|whitespace))+					>{ts = p;};
	name_in_double_quotes = (print - '"')+						>{ts = p;};
	name_in_single_quotes = (print - "'")+						>{ts = p;};
	tablename =  (print - ('#'|']'|'['|'"'|whitespace))+		>{ts = p;};
	tablename_in_double_quotes =  (print - '"')+				>{ts = p;};
	tablename_in_single_quotes =  (print - "'")+				>{ts = p;};

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen + 1) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, cur_line);
			fbreak;
		}

		while (ts[namelen] == ' ' || ts[namelen] == '\t')
			namelen--;

		name = strndup(ts, namelen + 1);
	}

	action saw_bool {
		struct toml_node node;

		node.type = TOML_BOOLEAN;
		node.value.integer = number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_int {
		char*				te = p;
		struct toml_node	node;

		node.type = TOML_INT;
		node.value.integer = negative ? -number : number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_float {
		char*				te = p;
		struct toml_node	node;

		floating = strtod(ts, &te);

		node.type = TOML_FLOAT;
		node.value.floating.value = floating;
		node.value.floating.precision = precision;

		if (!node.value.floating.precision && !exponent) {
			asprintf(&parse_error, "bad float\n");
			fbreak;
		}

		exponent = false;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_string {
		int					len = strp - string + 1;
		struct toml_node	node;

		*strp = 0;

		node.type = TOML_STRING;
		node.value.string = malloc(len);
		if (!node.value.string) {
			malloc_error = 1;
			fbreak;
		}
		memcpy(node.value.string, string, len);

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_date {
		char*	te = p;
		struct	toml_node node;

		node.type = TOML_DATE;
		node.value.rfc3339_time.epoch = timegm(&tm);
		node.value.rfc3339_time.offset_sign_negative = time_offset_is_negative;
		node.value.rfc3339_time.offset = time_offset;
		node.value.rfc3339_time.offset_is_zulu = time_offset_is_zulu;
		if (secfrac_ptr)
			node.value.rfc3339_time.sec_frac = strtol(secfrac_ptr, &te, 10);
		else
			node.value.rfc3339_time.sec_frac = -1;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), cur_line);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_inline_table {
		char*					tablename = name;
		struct toml_table_item*	item;
		struct toml_node*		place = cur_table ? cur_table : toml_root;
		bool					found = false;

		inline_table = true;

		list_for_each(&place->value.map, item, map) {
			if (strcmp(item->node.name, tablename) != 0)
				continue;

			found = true;
			break;
		}

		if (found)
		{
			asprintf(&parse_error, "duplicate entry %s\n", tablename);
			fbreak;
		}

		item = malloc(sizeof(*item));
		if (!item) {
			malloc_error = 1;
			fbreak;
		}

		item->node.name = strdup(tablename);
		item->node.type = TOML_TABLE;
		list_head_init(&item->node.value.map);
		list_add_tail(&place->value.map, &item->map);

		cur_table = &item->node;

		free(tablename);
	}

	action saw_table {
		char *ancestor, *tofree = NULL, *tablename;
		int item_added = 0;
		int len = (int)(p-ts);

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, len);

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", len, ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s", len, ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		int		ret;
		char*	tableName;

		tableName = strndup(ts, (int)(p-ts-1));
		ret = SawTableArray(toml_root, tableName, &cur_table, &parse_error);
		free(tableName);
		if (ret)
			fbreak;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	action bad_escape {
		asprintf(&parse_error, "bad escape \\%c", *p);
		fbreak;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			'#'			>{in_text = 0; fcall comment;}				->start			|
			[\t ]+		>{in_text = 0; indent = 0;} ${indent++;}	@{fgoto start;}	|
			[\n]		>{in_text = 0; cur_line++;}					@{fgoto start;}	|
			[\0]		>{in_text = 0; fbreak;}						@{fgoto start;}	|
			[^\t \n]	@{fhold;} %{in_text = 1;}					->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] ${cur_line++; fret;} ),

		# a table
		table: (
			tablename ']' @saw_table										->start	|
			'"' tablename_in_double_quotes '"' @saw_table ']'				->start	|
			"'" tablename_in_single_quotes "'" @saw_table ']'				->start	|
			'[' tablename ']' ']' @saw_table_array							->start	|
			'[' '"' tablename_in_double_quotes '"' ']' @saw_table_array ']'	->start	|
			'[' "'" tablename_in_single_quotes "'" ']' @saw_table_array ']'	->start
		),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]				-> basic_empty_or_multi_line	|
			[^"] ${fhold;}	-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]							-> basic_multi_line_start	|
			[^"] $saw_string ${fhold;}	-> start
		),

		basic_multi_line_start: (
			'\n' ${cur_line++;}	-> basic_multi_line	|
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${cur_line++;*strp++=fc;}	@{fgoto basic_multi_line;}	|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				@{fgoto basic_multi_line;}
		),

		basic_multi_line_escape: (
			[\n]	${cur_line++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${cur_line++;}	@{fgoto basic_multi_line_rm_ws;}	|
			[ \t]					@{fgoto basic_multi_line_rm_ws;}	|
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start						|
			[\n]		${cur_line++; *strp++=fc;}	@{fgoto basic_string_contents;}	|
			[\\]		${fcall str_escape;}		-> basic_string_contents		|
			[^"\n\\]	${*strp++=fc;}				@{fgoto basic_string_contents;}
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'"'	${*strp++='"';}			${fret;}	|
			'/'	${*strp++='/';}			${fret;}	|
			'\\'	${*strp++='\\';}	${fret;}	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0uU"/\\] $bad_escape
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			'_' ? digit ${number *= 10; number += fc-'0';}	@{fgoto number_or_date;}	|
			'-'	${tm.tm_year = number - 1900;}				->date						|
			[eE]											->exponent_part				|
			[.]	>{precision = 0;}							->fractional_part			|
			[\t ,\]\n\0] $saw_int ${fhold;}					->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9]	${precision++;}				@{fgoto fractional_part;}	|
			[eE]								->exponent_part				|
			[^0-9eE]	$saw_float ${fhold;}	->start
		),

		exponent_part: (
			[\-\+0-9]	${exponent=true;}		@{fgoto exponent_part;}	|
			[^\-\+0-9]	$saw_float ${fhold;}	->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: ( '' >{secfrac_ptr = NULL; time_offset = 0; time_offset_is_negative = 0; time_offset_is_zulu = 0;}
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);} -> fractional_second_or_offset
		),

		fractional_second_or_offset: (
			'.' digit* >{secfrac_ptr=p;} >{fhold;}	-> time_offset	|
			[^.] @{fhold;}							-> time_offset
		),

		time_offset: (
			('-' @{time_offset_is_negative=1;}|'+')
				digit{2} @{time_offset = atoi(fpc-1) * 60;}
				':'
				digit{2} @{time_offset += atoi(fpc-1);}
				@saw_date								-> start |
			'Z' >{time_offset_is_zulu = 1;} @saw_date	-> start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start	|
			[^']	${*strp++=fc;}	@{fgoto literal_string_contents;}
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${cur_line++;}	-> literal_multi_line	|
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote		|
			[\n]	${cur_line++;*strp++=fc;}	@{fgoto literal_multi_line;}	|
			[^'\n]	${*strp++=fc;}				@{fgoto literal_multi_line;}
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string							-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}	-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'		@{number = 1;}									-> true				|
			'false'		@{number = 0;}									-> false			|
			'"'			${strp = string;}								-> basic_string		|
			[']			${strp = string;}								-> literal_string	|
			('-'|'+')	${negative = fc == '-'; number = 0; ts = p;}	-> number_or_date	|
			digit		${negative = false; ts = p; number = fc-'0';}	-> number_or_date
		),

		# A list of values
		list: (
			'#'		>{fcall comment;}	->list			|
			'\n'	${cur_line++;}		@{fgoto list;}	|
			[\t ]						@{fgoto list;}	|
			','							@{fgoto list;}	|
			']'	$end_list				->start			|
			[^#\t, \n\]] ${fhold;}		->val
		),

		inline_table: (
			','								@{fgoto inline_table;}	|
			[\t ]							@{fgoto inline_table;}	|
			'}'	@{inline_table = false;}	-> start				|
			[^\t ,}] @{fhold;}				-> key
		),

		# A val can be either a list or a singular value
		val: (
			'#'				>{fcall comment;}	->val				|
			'\n'+			${ cur_line++; }	@{fgoto val;}		|
			[\t ]								@{fgoto val;}		|
			'['				$start_list			->list				|
			']'				$end_list			->start				|
			'{'				$saw_inline_table	->inline_table		|
			[^#\t \n[\]{]	${fhold;}			->singular
		),

		# A regular key
		key: (
			name @{namelen = (int)(p-ts);} whitespace '=' $saw_key								->val	|
			'"' name_in_double_quotes '"' @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val	|
			"'" name_in_single_quotes "'" @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'	>{fcall comment;}	->text			|
			'['						->table			|
			[\t ]					@{fgoto text;}	|
			'\n' ${cur_line++;}		->start			|
			[^#[\t \n]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node* toml_root, char* buf, int buflen)
{
	int indent = 0, cs, cur_line = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int precision;
	int namelen;
	int64_t number;
	bool negative;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];
	int in_text = 0;
	int time_offset = 0;
	char* secfrac_ptr;
	bool time_offset_is_negative = 0;
	bool time_offset_is_zulu = 0;
	bool inline_table = false;
	bool exponent = false;

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen + 1;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", cur_line);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, cur_line, p);
		free(parse_error);
		return 1;
	}

	if (in_text) {
		fprintf(stderr, "not in start, line %d\n", cur_line);
		return 1;
	}

	/* check we have consumed the entire buffer */
	if (p != pe) {
		fprintf(stderr, "entire buffer unconsumed, line %d\n", cur_line);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", cur_line, p);
		return 1;
	}

	return 0;
}

#include "toml.h"
#include "toml_private.h"

#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>
#include <math.h>
#include <signal.h>
#include <unicode/ustring.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static size_t
utf32ToUTF8(char* dst, int len, uint32_t utf32)
{
	if (utf32 < 0x80) {
		if (len < 1)
			return 0;

		*dst = (uint8_t)utf32;
		return 1;
	}

	if (utf32 < 0x000800)
	{
		if (len < 2)
			return 0;

		dst[0] = (uint8_t)(0xc0 | (utf32 >> 6));
		dst[1] = (uint8_t)(0x80 | (utf32 & 0x3f));
		return 2;
	}

	if (utf32 < 0x10000)
	{
		if (len < 3)
			return 0;

		dst[0] = (uint8_t)(0xE0 | (utf32 >> 12));
		dst[1] = (uint8_t)(0x80 | ((utf32 & 0x0FC0) >> 6));
		dst[2] = (uint8_t)(0x80 | (utf32 & 0x003F));
		return 3;
	}

	if (len < 4)
		return 0;

	dst[0] = (uint8_t)(0xF0 | (utf32 >> 18));
	dst[1] = (uint8_t)(0x80 | ((utf32 & 0x03F000) >> 12));
	dst[2] = (uint8_t)(0x80 | ((utf32 & 0x000FC0) >> 6));
	dst[3] = (uint8_t)(0x80 | (utf32 & 0x00003F));

	return 4;
}

bool add_node_to_tree(struct list_head* list_stack, struct toml_node* cur_table, struct toml_node* node, char* name, char** parse_error, int* malloc_error, int cur_line)
{
	struct toml_stack_item *cur_list =
					list_tail(list_stack, struct toml_stack_item, list);

	if (cur_list) {
		if (cur_list->list_type && cur_list->list_type != node->type) {
			asprintf(parse_error,
					"incompatible types list %s this %s line %d\n",
					toml_type_to_str(cur_list->list_type),
					toml_type_to_str(node->type), cur_line);
			return false;
		}
		cur_list->list_type = node->type;

		struct toml_list_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}

		memcpy(&item->node, node, sizeof(*node));
		item->node.name = NULL;
		list_add_tail(&cur_list->node->value.list, &item->list);
	} else {
		struct toml_table_item *item = malloc(sizeof(*item));
		if (!item) {
			*malloc_error = 1;
			return false;
		}
		memcpy(&item->node, node, sizeof(*node));
		item->node.name = name;
		list_add_tail(&cur_table->value.map, &item->map);
	}

	return true;
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - ('#'|'='|'"'|whitespace))+					>{ts = p;};
	name_in_double_quotes = (print - '"')+						>{ts = p;};
	name_in_single_quotes = (print - "'")+						>{ts = p;};
	tablename =  (print - ('#'|']'|'['|'"'|whitespace))+		>{ts = p;};
	tablename_in_double_quotes =  (print - '"')+				>{ts = p;};
	tablename_in_single_quotes =  (print - "'")+				>{ts = p;};

	action saw_key {
		struct toml_table_item* item = NULL;
		list_for_each(&cur_table->value.map, item, map) {
			if (strncmp(item->node.name, ts, namelen + 1) != 0)
				continue;

			asprintf(&parse_error,
					"duplicate key %s line %d\n",
					item->node.name, cur_line);
			fbreak;
		}

		while (ts[namelen] == ' ' || ts[namelen] == '\t')
			namelen--;

		name = strndup(ts, namelen + 1);
	}

	action saw_bool {
		struct toml_node node;

		node.type = TOML_BOOLEAN;
		node.value.integer = number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_int {
		char*				te = p;
		struct toml_node	node;

		node.type = TOML_INT;
		node.value.integer = negative ? -number : number;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_float {
		char*				te = p;
		struct toml_node	node;

		floating = strtod(ts, &te);

		node.type = TOML_FLOAT;
		node.value.floating.value = floating;
		node.value.floating.precision = precision;

		if (!node.value.floating.precision && !exponent) {
			asprintf(&parse_error, "bad float\n");
			fbreak;
		}

		exponent = false;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_string {
		int					len = strp - string + 1;
		struct toml_node	node;

		*strp = 0;

		node.type = TOML_STRING;
		node.value.string = malloc(len);
		if (!node.value.string) {
			malloc_error = 1;
			fbreak;
		}
		memcpy(node.value.string, string, len);

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_date {
		char*	te = p;
		struct	toml_node node;

		node.type = TOML_DATE;
		node.value.rfc3339_time.epoch = timegm(&tm);
		node.value.rfc3339_time.offset_sign_negative = time_offset_is_negative;
		node.value.rfc3339_time.offset = time_offset;
		node.value.rfc3339_time.offset_is_zulu = time_offset_is_zulu;
		if (secfrac_ptr)
			node.value.rfc3339_time.sec_frac = strtol(secfrac_ptr, &te, 10);
		else
			node.value.rfc3339_time.sec_frac = -1;

		if (!add_node_to_tree(&list_stack, cur_table, &node, name, &parse_error, &malloc_error, cur_line))
			fbreak;

		struct toml_stack_item *cur_list =
					list_tail(&list_stack, struct toml_stack_item, list);
		if (cur_list)
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the table
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_table_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the table */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_table->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item* tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			if (tail->list_type && tail->list_type != TOML_LIST) {
				asprintf(&parse_error,
						"incompatible types list %s this %s line %d\n",
						toml_type_to_str(tail->list_type),
						toml_type_to_str(TOML_BOOLEAN), cur_line);
				fbreak;
			}

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			tail->list_type = TOML_LIST;
			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
		else if (inline_table)
			fnext inline_table;
	}

	action saw_inline_table {
		char*					tablename = name;
		struct toml_table_item*	item;
		struct toml_node*		place = cur_table ? cur_table : toml_root;
		bool					found = false;

		inline_table = true;

		list_for_each(&place->value.map, item, map) {
			if (strcmp(item->node.name, tablename) != 0)
				continue;

			found = true;
			break;
		}

		if (found)
		{
			asprintf(&parse_error, "duplicate entry %s\n", tablename);
			fbreak;
		}

		item = malloc(sizeof(*item));
		if (!item) {
			malloc_error = 1;
			fbreak;
		}

		item->node.name = strdup(tablename);
		item->node.type = TOML_TABLE;
		list_head_init(&item->node.value.map);
		list_add_tail(&place->value.map, &item->map);

		cur_table = &item->node;

		free(tablename);
	}

	action saw_table {
		char *ancestor, *tofree = NULL, *tablename;
		int item_added = 0;
		int len = (int)(p-ts);

		struct toml_node *place = toml_root;

		tofree = tablename = strndup(ts, len);

		while ((ancestor = strsep(&tablename, "."))) {
			struct toml_table_item *item = NULL;
			int found = 0;

			if (strcmp(ancestor, "") == 0) {
				asprintf(&parse_error, "empty implicit table");
				fbreak;
			}

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_TABLE;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
			item_added = 1;
		}

		if (!item_added) {
			asprintf(&parse_error, "Duplicate item %.*s", len, ts);
			fbreak;
		}

		if (place->type != TOML_TABLE) {
			asprintf(&parse_error, "Attempt to overwrite table %.*s", len, ts);
			fbreak;
		}

		free(tofree);

		cur_table = place;
	}

	action saw_table_array {
		int		ret;
		char*	tableName;

		tableName = strndup(ts, (int)(p-ts-1));
		ret = SawTableArray(toml_root, tableName, &cur_table, &parse_error);
		free(tableName);
		if (ret)
			fbreak;
	}

	action saw_utf16 {
		UChar		utf16[2] = { 0 };
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		UErrorCode	err = U_ZERO_ERROR;
		char		utf16_str[5] = { 0 };

		memcpy(utf16_str, utf_start, 4);
		*utf16 = strtoul(utf16_str, NULL, 16);

		u_strToUTF8(strp, len, &outLen, utf16, 1, &err);
		strp += outLen;
		fret;
	}

	action saw_utf32 {
		uint32_t	utf32;
		int32_t		len = sizeof(string) - (strp - string);
		int32_t		outLen;
		char		utf32_str[9] = { 0 };

		memcpy(utf32_str, utf_start, 8);
		utf32 = strtoul(utf32_str, NULL, 16);

		outLen = utf32ToUTF8(strp, len, utf32);
		strp += outLen;
		fret;
	}

	action bad_escape {
		asprintf(&parse_error, "bad escape \\%c", *p);
		fbreak;
	}

	lines = (
		start: (
			# count the indentation to know where the tables end
			'#'			>{in_text = 0; fcall comment;}				->start	|
			[\t ]+		>{in_text = 0; indent = 0;} @{indent++;}	->start	|
			[\n]		>{in_text = 0; cur_line++;}					->start	|
			[\0]		>{in_text = 0; fbreak;}						->start	|
			[^\t \n]	@{fhold;} %{in_text = 1;}					->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] ${cur_line++; fret;} ),

		# a table
		table: (
			tablename ']' @saw_table										->start	|
			'"' tablename_in_double_quotes '"' @saw_table ']'				->start	|
			"'" tablename_in_single_quotes "'" @saw_table ']'				->start	|
			'[' tablename ']' ']' @saw_table_array							->start	|
			'[' '"' tablename_in_double_quotes '"' ']' @saw_table_array ']'	->start	|
			'[' "'" tablename_in_single_quotes "'" ']' @saw_table_array ']'	->start
		),

		# the boolean data type
		true:	( any	>{fhold;} $saw_bool	->start ),
		false:	( any	>{fhold;} $saw_bool	->start ),

		basic_string: (
			["]				-> basic_empty_or_multi_line	|
			[^"] ${fhold;}	-> basic_string_contents
		),

		basic_empty_or_multi_line: (
			["]							-> basic_multi_line_start	|
			[^"] $saw_string ${fhold;}	-> start
		),

		basic_multi_line_start: (
			'\n' ${cur_line++;}	-> basic_multi_line |
			[^\n] ${fhold;}		-> basic_multi_line
		),

		basic_multi_line: (
			["]										-> basic_multi_line_quote	|
			[\n]		${cur_line++;*strp++=fc;}	-> basic_multi_line			|
			[\\]									-> basic_multi_line_escape	|
			[^"\n\\]	${*strp++=fc;}				-> basic_multi_line
		),

		basic_multi_line_escape: (
			[\n]	${cur_line++;}			-> basic_multi_line_rm_ws	|
			[^\n]	${fcall str_escape;}	-> basic_multi_line
		),

		basic_multi_line_rm_ws: (
			[\n]	${cur_line++;}	-> basic_multi_line_rm_ws |
			[ \t]					-> basic_multi_line_rm_ws |
			[^ \t\n]	${fhold;}	-> basic_multi_line
		),

		basic_multi_line_quote: (
			["]								-> basic_multi_line_quote_2	|
			[^"]	${fhold;*strp++='"';}	-> basic_multi_line
		),

		basic_multi_line_quote_2: (
			["]		$saw_string							-> start |
			[^"]	${fhold;*strp++='"';*strp++='"';}	-> basic_multi_line
		),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		basic_string_contents: (
			'"'			$saw_string					-> start					|
			[\n]		${cur_line++; *strp++=fc;}	-> basic_string_contents	|
			[\\]		${fcall str_escape;}		-> basic_string_contents	|
			[^"\n\\]	${*strp++=fc;}				-> basic_string_contents
		),

		str_escape: (
			'b'	${*strp++=0x8;}			${fret;}	|
			't'	${*strp++='\t';}		${fret;}	|
			'n'	${*strp++='\n';}		${fret;}	|
			'f'	${*strp++=0xc;}			${fret;}	|
			'r'	${*strp++='\r';}		${fret;}	|
			'0'	${*strp++=0;}			${fret;}	|
			'"'	${*strp++='"';}			${fret;}	|
			'/'	${*strp++='/';}			${fret;}	|
			'\\'	${*strp++='\\';}	${fret;}	|
			'u'							-> unicode4	|
			'U'							-> unicode8	|
			[^btnfr0uU"/\\] $bad_escape
		),

		unicode4: (
			xdigit{4} >{utf_start=p;} @saw_utf16
		),

		unicode8: (
			xdigit{8} >{utf_start=p;} @saw_utf32
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			'_' ? digit ${number *= 10; number += fc-'0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}				->date				|
			[eE]											->exponent_part		|
			[.]	>{precision = 0;}							->fractional_part	|
			[\t ,\]\n\0] $saw_int ${fhold;}					->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9]	${precision++;}				->fractional_part	|
			[eE]								->exponent_part		|
			[^0-9eE]	$saw_float ${fhold;}	->start
		),

		exponent_part: (
			[\-\+0-9]	${exponent=true;}		->exponent_part	|
			[^\-\+0-9]	$saw_float ${fhold;}	->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: ( '' >{secfrac_ptr = NULL; time_offset = 0; time_offset_is_negative = 0; time_offset_is_zulu = 0;}
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);} -> fractional_second_or_offset
		),

		fractional_second_or_offset: (
			'.' digit* >{secfrac_ptr=p;} >{fhold;}	-> time_offset	|
			[^.] @{fhold;}							-> time_offset
		),

		time_offset: (
			('-' @{time_offset_is_negative=1;}|'+')
				digit{2} @{time_offset = atoi(fpc-1) * 60;}
				':'
				digit{2} @{time_offset += atoi(fpc-1);}
				@saw_date								-> start |
			'Z' >{time_offset_is_zulu = 1;} @saw_date	-> start
		),

		literal_string: (
			[']					-> literal_empty_or_multi_line		|
			[^']	${fhold;}	-> literal_string_contents
		),

		literal_string_contents: (
			[']		$saw_string		->start						|
			[^']	${*strp++=fc;}	->literal_string_contents
		),

		literal_empty_or_multi_line: (
			[']								-> literal_multi_line_start	|
			[^']	${fhold;} $saw_string	-> start
		),

		literal_multi_line_start: (
			'\n' ${cur_line++;}	-> literal_multi_line	|
			[^\n] ${fhold;}		-> literal_multi_line
		),

		literal_multi_line: (
			[']									-> literal_multi_line_quote	|
			[\n]	${cur_line++;*strp++=fc;}	-> literal_multi_line		|
			[^'\n]	${*strp++=fc;}				-> literal_multi_line
		),

		# saw 1 quote, if there's not another one go back to literalMultiLine
		literal_multi_line_quote: (
			[']									-> literal_multi_line_second_quote |
			[^']	${fhold;*strp++='\'';}		-> literal_multi_line
		),

		# saw 2 quotes, if there's not another one go back to literalMultiLine
		# if there is another then terminate the string
		literal_multi_line_second_quote: (
			[']		$saw_string							-> start				|
			[^']	${fhold;*strp++='\'';*strp++='\'';}	-> literal_multi_line
		),

		# Non-list value
		singular: (
			'true'		@{number = 1;}									-> true				|
			'false'		@{number = 0;}									-> false			|
			'"'			${strp = string;}								-> basic_string		|
			[']			${strp = string;}								-> literal_string	|
			('-'|'+')	${negative = fc == '-'; number = 0; ts = p;}	-> number_or_date	|
			digit		${negative = false; ts = p; number = fc-'0';}	-> number_or_date
		),

		# A list of values
		list: (
			'#'		>{fcall comment;}	->list	|
			'\n'	${cur_line++;}		->list	|
			[\t ]						->list	|
			','							->list	|
			']'	$end_list				->start	|
			[^#\t, \n\]] ${fhold;}		->val
		),

		inline_table: (
			','								-> inline_table |
			[\t ]							-> inline_table |
			'}'	@{inline_table = false;}	-> start		|
			[^\t ,}] @{fhold;}				-> key
		),

		# A val can be either a list or a singular value
		val: (
			'#'	>{fcall comment;}	->val			|
			'\n' ${ cur_line++; }	->val			|
			[\t ]					->val			|
			'['	$start_list			->list			|
			']'	$end_list			->start			|
			'{'	$saw_inline_table	->inline_table	|
			[^#\t \n[\]{] ${fhold;}	->singular
		),

		# A regular key
		key: (
			name @{namelen = (int)(p-ts);} whitespace '=' $saw_key								->val	|
			'"' name_in_double_quotes '"' @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val	|
			"'" name_in_single_quotes "'" @{namelen = (int)(p-ts-1);} whitespace '=' $saw_key	->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'	>{fcall comment;}	->text	|
			'['						->table	|
			[\t ]					->text	|
			'\n' ${cur_line++;}		->start	|
			[^#[\t \n]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node* toml_root, char* buf, int buflen)
{
	int indent = 0, cs, cur_line = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int precision;
	int namelen;
	int64_t number;
	bool negative;
	struct tm tm;
	double floating;
	char *name;
	char *parse_error = NULL;
	int malloc_error = 0;
	char* utf_start;
	int top = 0, stack[1024];
	int in_text = 0;
	int time_offset = 0;
	char* secfrac_ptr;
	bool time_offset_is_negative = 0;
	bool time_offset_is_zulu = 0;
	bool inline_table = false;
	bool exponent = false;

	struct toml_node *cur_table = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen + 1;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", cur_line);
		return 1;
	}

	if (parse_error) {
		fprintf(stderr, "%s at %d p = %.5s\n", parse_error, cur_line, p);
		free(parse_error);
		return 1;
	}

	if (in_text) {
		fprintf(stderr, "not in start, line %d\n", cur_line);
		return 1;
	}

	/* check we have consumed the entire buffer */
	if (p != pe) {
		fprintf(stderr, "entire buffer unconsumed, line %d\n", cur_line);
		return 1;
	}

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", cur_line, p);
		return 1;
	}

	return 0;
}


Delete test.rl

Delete test.rl


Let the lexer handle the prefix

Let the lexer handle the prefix

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )+;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \"HGVS\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s) }

cmd=ARGV.shift
run_lexer(cmd)

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \"HGVS\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)