compiles no

compiles no

// vim:syntax=ragel
#include <stdio.h>
#ifdef __PIKE__
# include "global.h"
# include "interpret.h"
# include "stralloc.h"
# include "mapping.h"
# include "svalue.h"
# include "array.h"
# include "module.h"
#else
# include <string.h>
# include <stdlib.h>
#endif

struct state {
    int cs, top;
};

struct depth {
    struct depth *up;
#ifdef __PIKE__
    struct svalue var;
#endif
    int level;
    int state;
};

%%{
    machine psyc;
    write data noerror nofinal;

    action begin {
	i = fpc;
    }

    action end {
	printf("stopping at %d\n", (int)fpc);
	printf("host: '%.*s' length: %d\n", (int)(fpc - i), i, fpc - i);
    }

    action descend {
	last = cur;
	cur->state = ftargs;
	printf("descending before state %d at '%c' (pos: %d).\n", ftargs, fc, (int)fpc);
	// TODO: use pikes alloc
	cur = (struct depth*)malloc(sizeof(struct depth));

	if (cur == NULL) {
#ifdef __PIKE__
	    Pike_error("malloc failed.");
#else
	    printf("malloc returned NULLLLLL!!!1!\n");
	    exit(-1);
#endif
	}
	cur->level = last->level+1;
	cur->up = last;
    }

    action free {
	free(last);
    }

    action ascend {
	last = cur;
	cur = cur->up;

	if (cur == NULL) {
#ifdef __PIKE__
	    Pike_error("pointer gone to heaven!");
#else
	    printf("pointer gone to heaven!\n");
	    exit(-1);
#endif
	}

	// keep the key, for gods sake!
#ifndef __PIKE__
	printf("ascending to %d at '%c' (pos: %d).\n", cur->state, fc, (int)fpc);
#endif
	fgoto *cur->state;
    }

    action begin_string {
#ifdef __PIKE__
	reset_string_builder(&s);
#else
	printf("beginning string.\n");
#endif
    }

    action mark { i = fpc; }
    action marknext { i = fpc + 1; }

    action string_append {
#ifdef __PIKE__
	if (fpc - i - 1 > 0) {
	    string_builder_binary_strcat(&s, i, (ptrdiff_t)(fpc - i - 1));
	}
#endif
    }

    action string_unquote {
#ifdef __PIKE__
	switch (fc) {
	case 'n':	string_builder_putchar(&s, '\n'); break;
	case 't':	string_builder_putchar(&s, '\t'); break;
	case 'r':	string_builder_putchar(&s, '\r'); break;
	case 'f':	string_builder_putchar(&s, '\f'); break;
	case 'b':	string_builder_putchar(&s, '\b'); break;
	case '"':
	case '\\':	string_builder_putchar(&s, fc); break;
	}
#endif
    }

    action end_string {
#ifdef __PIKE__
	cur->var.type = PIKE_T_STRING;
	cur->var.u.string = finish_string_builder(&s);
#else
	printf("string: '%.*s' length: %d\n", (int)(fpc - i), i, fpc - i);
#endif
    }

#	PIKE API:
#
#	string_builder_binary_strcat(string_builder *s, char *str, int lenth)
#
#	struct pike_string *finish_string_builder(struct string_builder *s)
#
#	void string_builder_append(struct string_builder *s,
#	                           PCHARP from,
#	                           ptrdiff_t len)
#
#	void string_builder_putchar(struct string_builder *s, int ch)
    string := (
	       start: (
		    '"' >string_append -> final |
		    '\\' >string_append -> unquote |
		    (any - ["\\]) -> start
		),
		unquote: (
		    [nt"\\/bfnrt] >string_unquote @marknext -> start
		)
    ) >mark >begin_string @end_string @ascend;

    action begin_mapping {
#ifdef __PIKE__
	cur->var.type = PIKE_T_MAPPING;
	cur->var.u.mapping = debug_allocate_mapping(8);
#else
	printf("beginning mapping.\n");
#endif
    }

    action end_mapping {
#ifndef __PIKE__
	printf("end mapping.\n");
#endif
    }

    action add_mapping {
#ifdef __PIKE__
	mapping_insert(cur->var.u.mapping, &key->var, &last->var);
#endif
	free(key);
	free(last);
    }

    alphtype char;
    access fsm.;

    myspace = ' ';

#	PIKE API
#
#	struct mapping *debug_allocate_mapping(int size)
#
#	mapping_insert(struct mapping *m,
#                        struct svalue *key,
#                       struct svalue *val)
    mapping := (
		  start: myspace* . (
		    '"' >descend >{ fgoto string; } . myspace* . ':' >{ key = last; } >descend >{ printf("value:\n"); fgoto value; } -> more |
		    '}' -> final
		  ),
		  more: myspace* . (
		    ',' -> start|
		    '}' -> final
		  ) >add_mapping
    ) >begin_mapping @ascend;

    action begin_number {
	printf("beginning number\n");
    }

    array := ( myspace* . (any - myspace) >descend >{ fhold; fgoto value;} . myspace* . (',' |  ']' >ascend ) )*;

    #json
    value = (myspace* . (
		  '"' >{ fgoto string; } |
		  '{' >{ fgoto mapping; } |
		  '[' >{ fgoto array; }
		 ) . myspace* );

    variable_name = '_' . (alpha | '_')+;
    variable = variable_name . '\t' . value >descend . '\n';

    main := variable* . variable_name . '\n' @{ done = 1; };
}%%

#ifdef __PIKE__
/*! @module Public
 */

/*! @module Parser
 */

/*! @module PSYC
 */

/*! @decl mapping parse(string s)
 *!
 *! Parses a JSON-formatted string and returns the corresponding mapping.
 */
PIKEFUN int parse(string data, object o) {
    char *p, *pe, *i;
    char done = 0;
    struct depth *cur, *last, *key;
    // we wont be building more than one string at once.
    struct string_builder s;
    struct state fsm;

    if (data->size_shift != 0) {
	Pike_error("Size shift != 0.");
	// no need to return. does a longjmp
    }

    // length n can be alot but is certainly enough.
    string_builder_allocate(&s, data->len, 0);

    p = (char*)STR0(data);
    pe = p + data->len;

    cur = (struct depth*)malloc(sizeof(struct depth));
    cur->level = 0;

    %%
    %% write init;
    %% write exec;

    free(cur);

    free_string_builder(&s);

    if ( done != 1 ) {
	RETURN (INT_TYPE)0;
    }

    // extract the mc.

    RETURN (INT_TYPE)1;
}

#else

int parse_psyc(char *d, struct state *f, int n) {
    char *p = d;
    char *pe = d + n;
    char *i = p;
    char done = 0;
    struct state fsm = *f;

    struct depth *cur, *last, *key;
    cur = (struct depth*)malloc(sizeof(struct depth));
    cur->level = 0;

    %% write init;
    %% write exec;

    printf("the machine stopped at char %c at position %d in state %d. left: %d, length: %d\n", *p, (int)p, fsm->cs, (int)(pe - p), strlen(p));
    free(cur);
    if ( done == 1 ) {
	printf("The machine parsed the packet successfully. data is: '%.*s'\n", pe - p, p);
    } else {
	printf("Error while parsing.");
    }

    return 0;
}

int main() {
    struct state fsm;

    char *one = "_sdlkf	\"hihihi\\\"\"\n_hallo	{ \"sdf\"  : \"sdf\", \"hihihahahiahia\\n\\f\" : \"\"}\n_hehe\t[ \"juchuu\" , \"wulle wulle wu\"   ]\n_message_public\nlksadsalkf ashdf kjhsafkj hsdakhf s\n\n\n\n";
    printf("%s\n", one);
    parse_psyc(one, &fsm, strlen(one));

    return 0;
}
#endif

// vim:syntax=ragel
#include <stdio.h>
#ifdef __PIKE__
# include "global.h"
# include "interpret.h"
# include "stralloc.h"
# include "mapping.h"
# include "svalue.h"
# include "array.h"
# include "module.h"
#else
# include <string.h>
# include <stdlib.h>
#endif

struct state {
    int cs, top;
};

struct depth {
    struct depth *up;
#ifdef __PIKE__
    struct svalue var;
#endif
    int level;
    int state;
};

%%{
    machine psyc;
    write data noerror nofinal;

    action begin {
	i = fpc;
    }

    action end {
	printf("stopping at %d\n", (int)fpc);
	printf("host: '%.*s' length: %d\n", (int)(fpc - i), i, fpc - i);
    }

    action descend {
	last = cur;
	cur->state = ftargs;
	printf("descending before state %d at '%c' (pos: %d).\n", ftargs, fc, (int)fpc);
	// TODO: use pikes alloc
	cur = (struct depth*)malloc(sizeof(struct depth));

	if (cur == NULL) {
#ifdef __PIKE__
	    Pike_error("malloc failed.");
#else
	    printf("malloc returned NULLLLLL!!!1!\n");
	    exit(-1);
#endif
	}
	cur->level = last->level+1;
	cur->up = last;
    }

    action free {
	free(last);
    }

    action ascend {
	last = cur;
	cur = cur->up;

	if (cur == NULL) {
#ifdef __PIKE__
	    Pike_error("pointer gone to heaven!");
#else
	    printf("pointer gone to heaven!\n");
	    exit(-1);
#endif
	}

	// keep the key, for gods sake!
#ifndef __PIKE__
	printf("ascending to %d at '%c' (pos: %d).\n", cur->state, fc, (int)fpc);
#endif
	fgoto *cur->state;
    }

    action begin_string {
#ifdef __PIKE__
	reset_string_builder(&s);
#else
	printf("beginning string.\n");
#endif
    }

    action mark { i = fpc; }
    action marknext { i = fpc + 1; }

    action string_append {
#ifdef __PIKE__
	if (fpc - i - 1 > 0) {
	    string_builder_binary_strcat(&s, i, (ptrdiff_t)(fpc - i - 1));
	}
#endif
    }

    action string_unquote {
#ifdef __PIKE__
	switch (fc) {
	case 'n':	string_builder_putchar(&s, '\n'); break;
	case 't':	string_builder_putchar(&s, '\t'); break;
	case 'r':	string_builder_putchar(&s, '\r'); break;
	case 'f':	string_builder_putchar(&s, '\f'); break;
	case 'b':	string_builder_putchar(&s, '\b'); break;
	case '"':
	case '\\':	string_builder_putchar(&s, fc); break;
	}
#endif
    }

    action end_string {
#ifdef __PIKE__
	cur->var.type = PIKE_T_STRING;
	cur->var.u.string = finish_string_builder(&s);
#else
	printf("string: '%.*s' length: %d\n", (int)(fpc - i), i, fpc - i);
#endif
    }

#	PIKE API:
#
#	string_builder_binary_strcat(string_builder *s, char *str, int lenth)
#
#	struct pike_string *finish_string_builder(struct string_builder *s)
#
#	void string_builder_append(struct string_builder *s,
#	                           PCHARP from,
#	                           ptrdiff_t len)
#
#	void string_builder_putchar(struct string_builder *s, int ch)
    string := (
	       start: (
		    '"' >string_append -> final |
		    '\\' >string_append -> unquote |
		    (any - ["\\]) -> start
		),
		unquote: (
		    [nt"\\/bfnrt] >string_unquote @marknext -> start
		)
    ) >mark >begin_string @end_string @ascend;

    action begin_mapping {
#ifdef __PIKE__
	cur->var.type = PIKE_T_MAPPING;
	cur->var.u.mapping = debug_allocate_mapping(8);
#else
	printf("beginning mapping.\n");
#endif
    }

    action end_mapping {
#ifndef __PIKE__
	printf("end mapping.\n");
#endif
    }

    action add_mapping {
#ifdef __PIKE__
	mapping_insert(cur->var.u.mapping, &key->var, &last->var);
#endif
	free(key);
	free(last);
    }

    alphtype char;
    access fsm->;

    myspace = ' ';

#	PIKE API
#
#	struct mapping *debug_allocate_mapping(int size)
#
#	mapping_insert(struct mapping *m,
#                        struct svalue *key,
#                       struct svalue *val)
    mapping := (
		  start: myspace* . (
		    '"' >descend >{ fgoto string; } . myspace* . ':' >{ key = last; } >descend >{ printf("value:\n"); fgoto value; } -> more |
		    '}' -> final
		  ),
		  more: myspace* . (
		    ',' -> start|
		    '}' -> final
		  ) >add_mapping
    ) >begin_mapping @ascend;

    action begin_number {
	printf("beginning number\n");
    }

    array := ( myspace* . (any - myspace) >descend >{ fhold; fgoto value;} . myspace* . (',' |  ']' >ascend ) )*;

    #json
    value = (myspace* . (
		  '"' >{ fgoto string; } |
		  '{' >{ fgoto mapping; } |
		  '[' >{ fgoto array; }
		 ) . myspace* );

    variable_name = '_' . (alpha | '_')+;
    variable = variable_name . '\t' . value >descend . '\n';

    main := variable* . variable_name . '\n' @{ done = 1; };
}%%

#ifdef __PIKE__
/*! @module Public
 */

/*! @module Parser
 */

/*! @module PSYC
 */

/*! @decl mapping parse(string s)
 *!
 *! Parses a JSON-formatted string and returns the corresponding mapping.
 */
PIKEFUN int parse(string data, object o) {
    char *p, *pe, *i;
    char done = 0;
    struct depth *cur, *last, *key;
    // we wont be building more than one string at once.
    struct string_builder s;

    if (stream->size_shift != 0) {
	Pike_error("Size shift != 0.");
	// no need to return. does a longjmp
    }

    // length n can be alot but is certainly enough.
    string_builder_allocate(&s, data->len, 0);

    p = (char*)STR0(data);
    pe = p + data->len;

    cur = (struct depth*)malloc(sizeof(struct depth));
    cur->level = 0;

    %% write init;
    %% write exec;

    free(cur);

    free_string_builder(&s);

    if ( done != 1 ) {
	RETURN (INT_TYPE)0;
    }

    // extract the mc.

    RETURN (INT_TYPE)1;
}

#else

int parse_psyc(char *d, struct state *fsm, int n) {
    char *p = d;
    char *pe = d + n;
    char *i = p;
    char done = 0;

    struct depth *cur, *last, *key;
    cur = (struct depth*)malloc(sizeof(struct depth));
    cur->level = 0;

    %% write init;
    %% write exec;

    printf("the machine stopped at char %c at position %d in state %d. left: %d, length: %d\n", *p, (int)p, fsm->cs, (int)(pe - p), strlen(p));
    free(cur);
    if ( done == 1 ) {
	printf("The machine parsed the packet successfully. data is: '%.*s'\n", pe - p, p);
    } else {
	printf("Error while parsing.");
    }

    return 0;
}

int main() {
    struct state fsm;

    char *one = "_sdlkf	\"hihihi\\\"\"\n_hallo	{ \"sdf\"  : \"sdf\", \"hihihahahiahia\\n\\f\" : \"\"}\n_hehe\t[ \"juchuu\" , \"wulle wulle wu\"   ]\n_message_public\nlksadsalkf ashdf kjhsafkj hsdakhf s\n\n\n\n";
    printf("%s\n", one);
    parse_psyc(one, &fsm, strlen(one));

    return 0;
}
#endif


Support \0 in double lit strings (patterns). refs #30.

/*
 *  Copyright 2006-2011 Adrian Thurston <thurston@complang.org>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "colm.h"
#include "lmscan.h"
#include "lmparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

//#define PRINT_TOKENS

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void Scanner::sectionParseInit()
{
	%% write init;
}

ostream &Scanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool Scanner::recursiveInclude( const char *inclFileName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 )
			return true;
	}
	return false;
}

void Scanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = ts;
	//cerr << "adding " << te - from << " to column" << endl;
	column += te - from;
	lastnl = 0;
}

void Scanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void Scanner::token( int type )
{
	token( type, 0, 0 );
}

bool isAbsolutePath( const char *path )
{
	return path[0] == '/';
}

ifstream *Scanner::tryOpenInclude( char **pathChecks, long &found )
{
	char **check = pathChecks;
	ifstream *inFile = new ifstream;

	while ( *check != 0 ) {
		inFile->open( *check );
		if ( inFile->is_open() ) {
			found = check - pathChecks;
			return inFile;
		}
		check += 1;
	}

	found = -1;
	delete inFile;
	return 0;
}

char **Scanner::makeIncludePathChecks( const char *thisFileName, const char *fileName )
{
	char **checks = 0;
	long nextCheck = 0;
	char *data = strdup(fileName);
	long length = strlen(fileName);

	/* Absolute path? */
	if ( isAbsolutePath( data ) ) {
		checks = new char*[2];
		checks[nextCheck++] = data;
	}
	else {
		/* Search from the the location of the current file. */
		checks = new char *[2 + includePaths.length()];
		const char *lastSlash = strrchr( thisFileName, '/' );
		if ( lastSlash == 0 )
			checks[nextCheck++] = data;
		else {
			long givenPathLen = (lastSlash - thisFileName) + 1;
			long checklen = givenPathLen + length;
			char *check = new char[checklen+1];
			memcpy( check, thisFileName, givenPathLen );
			memcpy( check+givenPathLen, data, length );
			check[checklen] = 0;
			checks[nextCheck++] = check;
		}

		/* Search from the include paths given on the command line. */
		for ( ArgsVector::Iter incp = includePaths; incp.lte(); incp++ ) {
			long pathLen = strlen( *incp );
			long checkLen = pathLen + 1 + length;
			char *check = new char[checkLen+1];
			memcpy( check, *incp, pathLen );
			check[pathLen] = '/';
			memcpy( check+pathLen+1, data, length );
			check[checkLen] = 0;
			checks[nextCheck++] = check;
		}
	}

	checks[nextCheck] = 0;
	return checks;
}

%%{
	machine section_parse;
	import "lmparse.h";

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_include
	{
		String src( lit, lit_len );
		String fileName;
		bool unused;

		/* Need a location. */
		InputLoc here;
		here.fileName = fileName;
		here.line = line;
		here.col = column;

		prepareLitString( fileName, unused, src, here );
		char **checks = makeIncludePathChecks( this->fileName, fileName );

		/* Open the input file for reading. */
		long found = 0;
		ifstream *inFile = tryOpenInclude( checks, found );
		if ( inFile == 0 ) {
			scan_error() << "include: could not open " <<
					fileName << " for reading" << endl;
		}
		else {
			/* Only proceed with the include if it was found. */
			if ( recursiveInclude( checks[found] ) )
				scan_error() << "include: this is a recursive include operation" << endl;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( checks[found] ) );

			Scanner scanner( fileName, *inFile, output, parser, includeDepth+1 );
			scanner.scan();
			delete inFile;

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );
		}
	}

	include_target =
		TK_Literal >clear_words @store_lit;

	include_stmt =
		( KW_Include include_target ) @handle_include
		<>err incl_err <>eof incl_err;

	action handle_token
	{
//	cout << Parser_lelNames[type] << " ";
//	if ( start != 0 ) {
//		cout.write( start, end-start );
//	}
//	cout << endl;

		InputLoc loc;

		#ifdef PRINT_TOKENS
		cerr << "scanner:" << line << ":" << column <<
				": sending token to the parser " << Parser_lelNames[*p];
		cerr << " " << toklen;
		if ( tokdata != 0 )
			cerr << " " << tokdata;
		cerr << endl;
		#endif

		loc.fileName = fileName;
		loc.line = line;
		loc.col = column;

		if ( tokdata != 0 && tokdata[toklen-1] == '\n' )
			loc.line -= 1;

		parser->token( loc, type, tokdata, toklen );
	}

	# Catch everything else.
	everything_else = ^( KW_Include ) @handle_token;

	main := (
		include_stmt |
		everything_else
	)*;
}%%

void Scanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;
	int *eof = 0;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();
}

void Scanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	/* Probably use: token( -1 ); */
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( TK_ReChar, '\0' ); };
		'\\a' => { token( TK_ReChar, '\a' ); };
		'\\b' => { token( TK_ReChar, '\b' ); };
		'\\t' => { token( TK_ReChar, '\t' ); };
		'\\n' => { token( TK_ReChar, '\n' ); };
		'\\v' => { token( TK_ReChar, '\v' ); };
		'\\f' => { token( TK_ReChar, '\f' ); };
		'\\r' => { token( TK_ReChar, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( TK_ReChar, ts+1, te ); };

		# Range dash in an OR expression.
		'-' => { token( TK_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( TK_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( TK_ReChar, ts, te ); };

	*|;

	regular_type := |*
		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		# Numbers
		number => { token( TK_UInt, ts, te ); };
		hex_number => { token( TK_Hex, ts, te ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, ts, te ); };

		'[' => { token( TK_SqOpen ); fcall or_literal; };
		'[^' => { token( TK_SqOpenNeg ); fcall or_literal; };

		'/' => { token( '/'); fret; };

		# Ignore.
		pound_comment => { updateCol(); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };

		':>'  => { token( TK_ColonGt ); };
		':>>' => { token( TK_ColonGtGt ); };
		'<:'  => { token( TK_LtColon ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;

	literal_pattern := |*
		'\\' '0' { litBuf.append( '\0' ); };
		'\\' 'a' { litBuf.append( '\a' ); };
		'\\' 'b' { litBuf.append( '\b' ); };
		'\\' 't' { litBuf.append( '\t' ); };
		'\\' 'n' { litBuf.append( '\n' ); };
		'\\' 'v' { litBuf.append( '\v' ); };
		'\\' 'f' { litBuf.append( '\f' ); };
		'\\' 'r' { litBuf.append( '\r' ); };

		'\\' any {
			litBuf.append( ts[1] );
		};
		'"' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '"' );
			fret;
		};
		NL => {
			litBuf.append( '\n' );
			token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
			litBuf.clear();
			token( '"' );
			fret;
		};
		'[' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '[' );
			fcall main;
		};
		any => {
			litBuf.append( *ts );
		};
	*|;

	# Parser definitions.
	main := |*
		'lex' => { token( KW_Lex ); };
		'commit' => { token( KW_Commit ); };
		'token' => { token( KW_Token ); };
		'literal' => { token( KW_Literal ); };
		'rl' => { token( KW_Rl ); };
		'def' => { token( KW_Def ); };
		'ignore' => { token( KW_Ignore ); };
		'construct' => { token( KW_Construct ); };
		'cons' => { token( KW_Construct ); };
		'new' => { token( KW_New ); };
		'if' => { token( KW_If ); };
		'reject' => { token( KW_Reject ); };
		'while' => { token( KW_While ); };
		'else' => { token( KW_Else ); };
		'elsif' => { token( KW_Elsif ); };
		'match' => { token( KW_Match ); };
		'for' => { token( KW_For ); };
		'iter' => { token( KW_Iter ); };
		'prints' => { token( KW_PrintStream ); };
		'print' => { token( KW_Print ); };
		'print_xml_ac' => { token( KW_PrintXMLAC ); };
		'print_xml' => { token( KW_PrintXML ); };
		'namespace' => { token( KW_Namespace ); };
		'lex' => { token( KW_Lex ); };
		'map' => { token( KW_Map ); };
		'list' => { token( KW_List ); };
		'vector' => { token( KW_Vector ); };
		'accum' => { token( KW_Accum ); };
		'parser' => { token( KW_Accum ); };
		'return' => { token( KW_Return ); };
		'break' => { token( KW_Break ); };
		'yield' => { token( KW_Yield ); };
		'typeid' => { token( KW_TypeId ); };
		'make_token' => { token( KW_MakeToken ); };
		'make_tree' => { token( KW_MakeTree ); };
		'reducefirst' => { token( KW_ReduceFirst ); };
		'for' => { token( KW_For ); };
		'in' => { token( KW_In ); };
		'nil' => { token( KW_Nil ); };
		'true' => { token( KW_True ); };
		'false' => { token( KW_False ); };
		'parse' => { token( KW_Parse ); };
		'parse_stop' => { token( KW_ParseStop ); };
		'global' => { token( KW_Global ); };
		'ptr' => { token( KW_Ptr ); };
		'ref' => { token( KW_Ref ); };
		'deref' => { token( KW_Deref ); };
		'require' => { token( KW_Require ); };
		'preeof' => { token( KW_Preeof ); };
		'left' => { token( KW_Left ); };
		'right' => { token( KW_Right ); };
		'nonassoc' => { token( KW_Nonassoc ); };
		'prec' => { token( KW_Prec ); };
		'include' => {token( KW_Include ); };
		'context' => {token( KW_Context ); };
		'alias' => {token( KW_Alias ); };

		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		number => { token( TK_Number, ts, te ); };

		'/' => {
			token( '/' );
			if ( parser->enterRl )
				fcall regular_type;
		};

		"~" [^\n]* NL => {
			token( '"' );
			token( TK_LitPat, ts+1, te );
			token( '"' );
		};

		"'" ([^'\\\n] | '\\' (any | NL))* ( "'" | NL ) => {
			token( TK_Literal, ts, te );
		};

		'"' => {
			token( '"' );
			litBuf.clear();
			fcall literal_pattern;
		};
		'[' => {
			token( '[' );
			fcall main;
		};

		']' => {
			token( ']' );
			if ( top > 0 )
				fret;
		};

		# Ignore.
		pound_comment => { updateCol(); };

		'=>' => { token( TK_DoubleArrow ); };
		'==' => { token( TK_DoubleEql ); };
		'!=' => { token( TK_NotEql ); };
		'::' => { token( TK_DoubleColon ); };
		'<=' => { token( TK_LessEql ); };
		'>=' => { token( TK_GrtrEql ); };
		'->' => { token( TK_RightArrow ); };
		'&&' => { token( TK_AmpAmp ); };
		'||' => { token( TK_BarBar ); };
		'<<' => { token( TK_LtLt ); };

		('+' | '-' | '*' | '/' | '(' | ')' | '@' | '$' ) => { token( *ts ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;
}%%

%% write data;

void Scanner::scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[32];
	bool execute = true;

	sectionParseInit();
	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( ts != 0 )
				ts = newbuf + ( ts - buf );
			te = newbuf + ( te - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		char *eof = 0;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "colm scanner error (metalanguage)" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = ts;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( ts != 0 )
				ts -= shiftback;
			te -= shiftback;

			preserve = buf;
		}
	}
	delete[] buf;
}

void Scanner::eof()
{
	InputLoc loc;
	loc.fileName = "<EOF>";
	loc.line = line;
	loc.col = 1;
	parser->token( loc, Parser_tk_eof, 0, 0 );
}

/*
 *  Copyright 2006-2011 Adrian Thurston <thurston@complang.org>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "colm.h"
#include "lmscan.h"
#include "lmparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

//#define PRINT_TOKENS

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void Scanner::sectionParseInit()
{
	%% write init;
}

ostream &Scanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool Scanner::recursiveInclude( const char *inclFileName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 )
			return true;
	}
	return false;
}

void Scanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = ts;
	//cerr << "adding " << te - from << " to column" << endl;
	column += te - from;
	lastnl = 0;
}

void Scanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void Scanner::token( int type )
{
	token( type, 0, 0 );
}

bool isAbsolutePath( const char *path )
{
	return path[0] == '/';
}

ifstream *Scanner::tryOpenInclude( char **pathChecks, long &found )
{
	char **check = pathChecks;
	ifstream *inFile = new ifstream;

	while ( *check != 0 ) {
		inFile->open( *check );
		if ( inFile->is_open() ) {
			found = check - pathChecks;
			return inFile;
		}
		check += 1;
	}

	found = -1;
	delete inFile;
	return 0;
}

char **Scanner::makeIncludePathChecks( const char *thisFileName, const char *fileName )
{
	char **checks = 0;
	long nextCheck = 0;
	char *data = strdup(fileName);
	long length = strlen(fileName);

	/* Absolute path? */
	if ( isAbsolutePath( data ) ) {
		checks = new char*[2];
		checks[nextCheck++] = data;
	}
	else {
		/* Search from the the location of the current file. */
		checks = new char *[2 + includePaths.length()];
		const char *lastSlash = strrchr( thisFileName, '/' );
		if ( lastSlash == 0 )
			checks[nextCheck++] = data;
		else {
			long givenPathLen = (lastSlash - thisFileName) + 1;
			long checklen = givenPathLen + length;
			char *check = new char[checklen+1];
			memcpy( check, thisFileName, givenPathLen );
			memcpy( check+givenPathLen, data, length );
			check[checklen] = 0;
			checks[nextCheck++] = check;
		}

		/* Search from the include paths given on the command line. */
		for ( ArgsVector::Iter incp = includePaths; incp.lte(); incp++ ) {
			long pathLen = strlen( *incp );
			long checkLen = pathLen + 1 + length;
			char *check = new char[checkLen+1];
			memcpy( check, *incp, pathLen );
			check[pathLen] = '/';
			memcpy( check+pathLen+1, data, length );
			check[checkLen] = 0;
			checks[nextCheck++] = check;
		}
	}

	checks[nextCheck] = 0;
	return checks;
}

%%{
	machine section_parse;
	import "lmparse.h";

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_include
	{
		String src( lit, lit_len );
		String fileName;
		bool unused;

		/* Need a location. */
		InputLoc here;
		here.fileName = fileName;
		here.line = line;
		here.col = column;

		prepareLitString( fileName, unused, src, here );
		char **checks = makeIncludePathChecks( this->fileName, fileName );

		/* Open the input file for reading. */
		long found = 0;
		ifstream *inFile = tryOpenInclude( checks, found );
		if ( inFile == 0 ) {
			scan_error() << "include: could not open " <<
					fileName << " for reading" << endl;
		}
		else {
			/* Only proceed with the include if it was found. */
			if ( recursiveInclude( checks[found] ) )
				scan_error() << "include: this is a recursive include operation" << endl;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( checks[found] ) );

			Scanner scanner( fileName, *inFile, output, parser, includeDepth+1 );
			scanner.scan();
			delete inFile;

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );
		}
	}

	include_target =
		TK_Literal >clear_words @store_lit;

	include_stmt =
		( KW_Include include_target ) @handle_include
		<>err incl_err <>eof incl_err;

	action handle_token
	{
//	cout << Parser_lelNames[type] << " ";
//	if ( start != 0 ) {
//		cout.write( start, end-start );
//	}
//	cout << endl;

		InputLoc loc;

		#ifdef PRINT_TOKENS
		cerr << "scanner:" << line << ":" << column <<
				": sending token to the parser " << Parser_lelNames[*p];
		cerr << " " << toklen;
		if ( tokdata != 0 )
			cerr << " " << tokdata;
		cerr << endl;
		#endif

		loc.fileName = fileName;
		loc.line = line;
		loc.col = column;

		if ( tokdata != 0 && tokdata[toklen-1] == '\n' )
			loc.line -= 1;

		parser->token( loc, type, tokdata, toklen );
	}

	# Catch everything else.
	everything_else = ^( KW_Include ) @handle_token;

	main := (
		include_stmt |
		everything_else
	)*;
}%%

void Scanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;
	int *eof = 0;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();
}

void Scanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	/* Probably use: token( -1 ); */
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( TK_ReChar, '\0' ); };
		'\\a' => { token( TK_ReChar, '\a' ); };
		'\\b' => { token( TK_ReChar, '\b' ); };
		'\\t' => { token( TK_ReChar, '\t' ); };
		'\\n' => { token( TK_ReChar, '\n' ); };
		'\\v' => { token( TK_ReChar, '\v' ); };
		'\\f' => { token( TK_ReChar, '\f' ); };
		'\\r' => { token( TK_ReChar, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( TK_ReChar, ts+1, te ); };

		# Range dash in an OR expression.
		'-' => { token( TK_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( TK_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( TK_ReChar, ts, te ); };

	*|;

	regular_type := |*
		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		# Numbers
		number => { token( TK_UInt, ts, te ); };
		hex_number => { token( TK_Hex, ts, te ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, ts, te ); };

		'[' => { token( TK_SqOpen ); fcall or_literal; };
		'[^' => { token( TK_SqOpenNeg ); fcall or_literal; };

		'/' => { token( '/'); fret; };

		# Ignore.
		pound_comment => { updateCol(); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };

		':>'  => { token( TK_ColonGt ); };
		':>>' => { token( TK_ColonGtGt ); };
		'<:'  => { token( TK_LtColon ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;

	literal_pattern := |*
		'\\' 'a' { litBuf.append( '\a' ); };
		'\\' 'b' { litBuf.append( '\b' ); };
		'\\' 't' { litBuf.append( '\t' ); };
		'\\' 'n' { litBuf.append( '\n' ); };
		'\\' 'v' { litBuf.append( '\v' ); };
		'\\' 'f' { litBuf.append( '\f' ); };
		'\\' 'r' { litBuf.append( '\r' ); };

		'\\' any {
			litBuf.append( ts[1] );
		};
		'"' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '"' );
			fret;
		};
		NL => {
			litBuf.append( '\n' );
			token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
			litBuf.clear();
			token( '"' );
			fret;
		};
		'[' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '[' );
			fcall main;
		};
		any => {
			litBuf.append( *ts );
		};
	*|;

	# Parser definitions.
	main := |*
		'lex' => { token( KW_Lex ); };
		'commit' => { token( KW_Commit ); };
		'token' => { token( KW_Token ); };
		'literal' => { token( KW_Literal ); };
		'rl' => { token( KW_Rl ); };
		'def' => { token( KW_Def ); };
		'ignore' => { token( KW_Ignore ); };
		'construct' => { token( KW_Construct ); };
		'cons' => { token( KW_Construct ); };
		'new' => { token( KW_New ); };
		'if' => { token( KW_If ); };
		'reject' => { token( KW_Reject ); };
		'while' => { token( KW_While ); };
		'else' => { token( KW_Else ); };
		'elsif' => { token( KW_Elsif ); };
		'match' => { token( KW_Match ); };
		'for' => { token( KW_For ); };
		'iter' => { token( KW_Iter ); };
		'prints' => { token( KW_PrintStream ); };
		'print' => { token( KW_Print ); };
		'print_xml_ac' => { token( KW_PrintXMLAC ); };
		'print_xml' => { token( KW_PrintXML ); };
		'namespace' => { token( KW_Namespace ); };
		'lex' => { token( KW_Lex ); };
		'map' => { token( KW_Map ); };
		'list' => { token( KW_List ); };
		'vector' => { token( KW_Vector ); };
		'accum' => { token( KW_Accum ); };
		'parser' => { token( KW_Accum ); };
		'return' => { token( KW_Return ); };
		'break' => { token( KW_Break ); };
		'yield' => { token( KW_Yield ); };
		'typeid' => { token( KW_TypeId ); };
		'make_token' => { token( KW_MakeToken ); };
		'make_tree' => { token( KW_MakeTree ); };
		'reducefirst' => { token( KW_ReduceFirst ); };
		'for' => { token( KW_For ); };
		'in' => { token( KW_In ); };
		'nil' => { token( KW_Nil ); };
		'true' => { token( KW_True ); };
		'false' => { token( KW_False ); };
		'parse' => { token( KW_Parse ); };
		'parse_stop' => { token( KW_ParseStop ); };
		'global' => { token( KW_Global ); };
		'ptr' => { token( KW_Ptr ); };
		'ref' => { token( KW_Ref ); };
		'deref' => { token( KW_Deref ); };
		'require' => { token( KW_Require ); };
		'preeof' => { token( KW_Preeof ); };
		'left' => { token( KW_Left ); };
		'right' => { token( KW_Right ); };
		'nonassoc' => { token( KW_Nonassoc ); };
		'prec' => { token( KW_Prec ); };
		'include' => {token( KW_Include ); };
		'context' => {token( KW_Context ); };
		'alias' => {token( KW_Alias ); };

		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		number => { token( TK_Number, ts, te ); };

		'/' => {
			token( '/' );
			if ( parser->enterRl )
				fcall regular_type;
		};

		"~" [^\n]* NL => {
			token( '"' );
			token( TK_LitPat, ts+1, te );
			token( '"' );
		};

		"'" ([^'\\\n] | '\\' (any | NL))* ( "'" | NL ) => {
			token( TK_Literal, ts, te );
		};

		'"' => {
			token( '"' );
			litBuf.clear();
			fcall literal_pattern;
		};
		'[' => {
			token( '[' );
			fcall main;
		};

		']' => {
			token( ']' );
			if ( top > 0 )
				fret;
		};

		# Ignore.
		pound_comment => { updateCol(); };

		'=>' => { token( TK_DoubleArrow ); };
		'==' => { token( TK_DoubleEql ); };
		'!=' => { token( TK_NotEql ); };
		'::' => { token( TK_DoubleColon ); };
		'<=' => { token( TK_LessEql ); };
		'>=' => { token( TK_GrtrEql ); };
		'->' => { token( TK_RightArrow ); };
		'&&' => { token( TK_AmpAmp ); };
		'||' => { token( TK_BarBar ); };
		'<<' => { token( TK_LtLt ); };

		('+' | '-' | '*' | '/' | '(' | ')' | '@' | '$' ) => { token( *ts ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;
}%%

%% write data;

void Scanner::scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[32];
	bool execute = true;

	sectionParseInit();
	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( ts != 0 )
				ts = newbuf + ( ts - buf );
			te = newbuf + ( te - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		char *eof = 0;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "colm scanner error (metalanguage)" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = ts;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( ts != 0 )
				ts -= shiftback;
			te -= shiftback;

			preserve = buf;
		}
	}
	delete[] buf;
}

void Scanner::eof()
{
	InputLoc loc;
	loc.fileName = "<EOF>";
	loc.line = line;
	loc.col = 1;
	parser->token( loc, Parser_tk_eof, 0, 0 );
}


began commenting mtoc

began commenting mtoc

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # line in documentation block
  docu_block_line :=
    (
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     EOL @(add_block_line) @{line++;}
     |
     '"""' @(add_block_line) @{fret;}
     |
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down
    ('}') @{ go_level_down(); fret; }
    )*;

  main := (
           MWSOC
           | (
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string ist matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // check wether str matches glob
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename ist matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

// constructor
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "doxygen/mtoc.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "Error opening configuration file '" << conffilename << "'\n";
    exit(-2);
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    // string for temporary tokens
    string tmp_string;
    // spare space in buffer
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    // read configuration file chunk in buffer
    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  EOF = 0;

  EOL = '\r'? . '\n';

  default = ^0;

  IDENT = [A-Z]+;

  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  GLOB = (default - [\n \t;\{])+;

  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  ENDRULE = ';' @{arg_to_be_added_ = false;};

  docu_block_line :=
    (
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     EOL @(add_block_line) @{line++;}
     |
     '"""' @(add_block_line) @{fret;}
     |
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down
    ('}') @{ go_level_down(); fret; }
    )*;

  main := (
           MWSOC
           | (
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string ist matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // check wether str matches glob
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename ist matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
    return true;
/*  if(l == level_)
  {*/
    /* true if all leading directories in path or the whole path string are
     * matched by pattern
     */
/*    found = s.rfind("/");
    if(found != string::npos)
    {
      str = s.substr(0, found);
      if(check_for_match(l, str.c_str()))
        return true;
    }
    // else: try to match complete string
    return check_for_match(l, s.c_str());
  }*/

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

// constructor
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "doxygen/mtoc.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "Error opening configuration file '" << conffilename << "'\n";
    exit(-2);
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    // string for temporary tokens
    string tmp_string;
    // spare space in buffer
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    // read configuration file chunk in buffer
    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */