Fix compiler warnings.

Fix compiler warnings.

/*
 * redcloth_scan.c.rl
 *
 * Copyright (C) 2009 Jason Garber
 */
#define redcloth_scan_c

#include <ruby.h>
#include "redcloth.h"

VALUE mRedCloth, super_ParseError, super_RedCloth, super_HTML, super_LATEX;
VALUE SYM_escape_preformatted, SYM_escape_attributes;

%%{

  machine redcloth_scan;
  include redcloth_common "redcloth_common.c.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  include redcloth_scan "redcloth_scan.rl";

}%%

%% write data nofinal;

VALUE
redcloth_transform(self, p, pe, refs)
  VALUE self;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest = 0;
  char *ts = NULL, *te = NULL, *reg = NULL, *bck = NULL, *attr_reg = NULL, *eof = NULL;
  VALUE html = STR_NEW2("");
  VALUE table = STR_NEW2("");
  VALUE block = STR_NEW2("");
  VALUE regs, attr_regs; CLEAR_REGS()

  VALUE list_layout = Qnil;
  const char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  VALUE plain_block; SET_PLAIN_BLOCK("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING_LEN(block) > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return redcloth_transform(self, orig_p, orig_pe, refs_found);
  } else {
    rb_funcall(self, rb_intern("after_transform"), 1, html);
    return html;
  }
}

VALUE
redcloth_transform2(self, str)
  VALUE self, str;
{
  StringValue(str);
  rb_funcall(self, rb_intern("before_transform"), 1, str);
  return redcloth_transform(self, RSTRING_PTR(str), RSTRING_PTR(str) + RSTRING_LEN(str) + 1, Qnil);
}

/*
 * Converts special characters into HTML entities.
 */
static VALUE
redcloth_html_esc(int argc, VALUE* argv, VALUE self) //(self, str, level)
{
  VALUE str, level;

  rb_scan_args(argc, argv, "11", &str, &level);

  VALUE new_str = STR_NEW2("");
  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts;
  const char *ch = NULL;
  if (te <= ts) return Qnil;

  while (t2 < te) {
    ch = NULL;

    // normal + pre
    switch (*t2)
    {
      case '&':  ch = "amp";    break;
      case '>':  ch = "gt";     break;
      case '<':  ch = "lt";     break;
    }

    // normal (non-pre)
    if (level != SYM_escape_preformatted) {
      switch (*t2)
      {
        case '\n': ch = "br";     break;
        case '"' : ch = "quot";   break;
        case '\'':
          ch = (level == SYM_escape_attributes) ? "apos" : "squot";
          break;
      }
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      rb_str_concat(new_str, rb_funcall(self, rb_intern(ch), 1, rb_hash_new()));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Converts special characters into LaTeX entities.
 */
static VALUE
redcloth_latex_esc(VALUE self, VALUE str)
{
  VALUE new_str = STR_NEW2("");

  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts;
  const char *ch = NULL;
  if (te <= ts) return Qnil;

  while (t2 < te) {
    ch = NULL;

    switch (*t2)
    {
      case '{':  ch = "#123";   break;
      case '}':  ch = "#125";   break;
      case '\\': ch = "#92";    break;
      case '#':  ch = "#35";    break;
      case '$':  ch = "#36";    break;
      case '%':  ch = "#37";    break;
      case '&':  ch = "amp";    break;
      case '_':  ch = "#95";    break;
      case '^':  ch = "circ";   break;
      case '~':  ch = "tilde";  break;
      case '<':  ch = "lt";     break;
      case '>':  ch = "gt";     break;
      case '\n': ch = "#10";    break;
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      VALUE opts = rb_hash_new();
      rb_hash_aset(opts, ID2SYM(rb_intern("text")), STR_NEW2(ch));
      rb_str_concat(new_str, rb_funcall(self, rb_intern("entity"), 1, opts));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Transforms a Textile document with +formatter+
 */
static VALUE
redcloth_to(self, formatter)
  VALUE self, formatter;
{
  rb_funcall(self, rb_intern("delete!"), 1, STR_NEW2("\r"));
  VALUE working_copy = rb_obj_clone(self);
  rb_extend_object(working_copy, formatter);

  if (rb_funcall(working_copy, rb_intern("lite_mode"), 0) == Qtrue) {
    return redcloth_inline2(working_copy, self, rb_hash_new());
  } else {
    return redcloth_transform2(working_copy, self);
  }
}

void Init_redcloth_scan()
{
  mRedCloth = rb_define_module("RedCloth");
  rb_define_const( mRedCloth, "EXTENSION_LANGUAGE", rb_str_new2("C") );
  /* A Textile document that can be converted to other formats. See
   the README for Textile syntax. */
  super_RedCloth = rb_define_class_under(mRedCloth, "TextileDoc", rb_cString);
  rb_define_method(super_RedCloth, "to", redcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  /* Escaping */
  rb_define_method(super_RedCloth, "html_esc", redcloth_html_esc, -1);
  rb_define_method(super_RedCloth, "latex_esc", redcloth_latex_esc, 1);
  SYM_escape_preformatted   = ID2SYM(rb_intern("html_escape_preformatted"));
  SYM_escape_attributes     = ID2SYM(rb_intern("html_escape_attributes"));
}

/*
 * redcloth_scan.c.rl
 *
 * Copyright (C) 2009 Jason Garber
 */
#define redcloth_scan_c

#include <ruby.h>
#include "redcloth.h"

VALUE mRedCloth, super_ParseError, super_RedCloth, super_HTML, super_LATEX;
int SYM_escape_preformatted, SYM_escape_attributes;

%%{

  machine redcloth_scan;
  include redcloth_common "redcloth_common.c.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  include redcloth_scan "redcloth_scan.rl";

}%%

%% write data nofinal;

VALUE
redcloth_transform(self, p, pe, refs)
  VALUE self;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest = 0;
  char *ts = NULL, *te = NULL, *reg = NULL, *bck = NULL, *attr_reg = NULL, *eof = NULL;
  VALUE html = STR_NEW2("");
  VALUE table = STR_NEW2("");
  VALUE block = STR_NEW2("");
  VALUE regs, attr_regs; CLEAR_REGS()

  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  VALUE plain_block; SET_PLAIN_BLOCK("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING_LEN(block) > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return redcloth_transform(self, orig_p, orig_pe, refs_found);
  } else {
    rb_funcall(self, rb_intern("after_transform"), 1, html);
    return html;
  }
}

VALUE
redcloth_transform2(self, str)
  VALUE self, str;
{
  StringValue(str);
  rb_funcall(self, rb_intern("before_transform"), 1, str);
  return redcloth_transform(self, RSTRING_PTR(str), RSTRING_PTR(str) + RSTRING_LEN(str) + 1, Qnil);
}

/*
 * Converts special characters into HTML entities.
 */
static VALUE
redcloth_html_esc(int argc, VALUE* argv, VALUE self) //(self, str, level)
{
  VALUE str, level;

  rb_scan_args(argc, argv, "11", &str, &level);

  VALUE new_str = STR_NEW2("");
  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return Qnil;

  while (t2 < te) {
    ch = NULL;

    // normal + pre
    switch (*t2)
    {
      case '&':  ch = "amp";    break;
      case '>':  ch = "gt";     break;
      case '<':  ch = "lt";     break;
    }

    // normal (non-pre)
    if (level != SYM_escape_preformatted) {
      switch (*t2)
      {
        case '\n': ch = "br";     break;
        case '"' : ch = "quot";   break;
        case '\'':
          ch = (level == SYM_escape_attributes) ? "apos" : "squot";
          break;
      }
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      rb_str_concat(new_str, rb_funcall(self, rb_intern(ch), 1, rb_hash_new()));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Converts special characters into LaTeX entities.
 */
static VALUE
redcloth_latex_esc(VALUE self, VALUE str)
{
  VALUE new_str = STR_NEW2("");

  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return Qnil;

  while (t2 < te) {
    ch = NULL;

    switch (*t2)
    {
      case '{':  ch = "#123";   break;
      case '}':  ch = "#125";   break;
      case '\\': ch = "#92";    break;
      case '#':  ch = "#35";    break;
      case '$':  ch = "#36";    break;
      case '%':  ch = "#37";    break;
      case '&':  ch = "amp";    break;
      case '_':  ch = "#95";    break;
      case '^':  ch = "circ";   break;
      case '~':  ch = "tilde";  break;
      case '<':  ch = "lt";     break;
      case '>':  ch = "gt";     break;
      case '\n': ch = "#10";    break;
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      VALUE opts = rb_hash_new();
      rb_hash_aset(opts, ID2SYM(rb_intern("text")), STR_NEW2(ch));
      rb_str_concat(new_str, rb_funcall(self, rb_intern("entity"), 1, opts));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Transforms a Textile document with +formatter+
 */
static VALUE
redcloth_to(self, formatter)
  VALUE self, formatter;
{
  rb_funcall(self, rb_intern("delete!"), 1, STR_NEW2("\r"));
  VALUE working_copy = rb_obj_clone(self);
  rb_extend_object(working_copy, formatter);

  if (rb_funcall(working_copy, rb_intern("lite_mode"), 0) == Qtrue) {
    return redcloth_inline2(working_copy, self, rb_hash_new());
  } else {
    return redcloth_transform2(working_copy, self);
  }
}

void Init_redcloth_scan()
{
  mRedCloth = rb_define_module("RedCloth");
  rb_define_const( mRedCloth, "EXTENSION_LANGUAGE", rb_str_new2("C") );
  /* A Textile document that can be converted to other formats. See
   the README for Textile syntax. */
  super_RedCloth = rb_define_class_under(mRedCloth, "TextileDoc", rb_cString);
  rb_define_method(super_RedCloth, "to", redcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  /* Escaping */
  rb_define_method(super_RedCloth, "html_esc", redcloth_html_esc, -1);
  rb_define_method(super_RedCloth, "latex_esc", redcloth_latex_esc, 1);
  SYM_escape_preformatted   = ID2SYM(rb_intern("html_escape_preformatted"));
  SYM_escape_attributes     = ID2SYM(rb_intern("html_escape_attributes"));
}


Ragel works

Ragel works

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  integer     = ('+'|'-')?[0-9]+;
  float       = ('+'|'-')?[0-9]+'.'[0-9]+;
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+;
  string      = ["][^"]*["];
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') %{ puts "**keyword" } );
  value       = ( (integer|float|boolean|identifier|string) %{ puts "***value" } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

  # key := (

    # integer => {
    #   emit(:integer_literal, data, token_array, ts, te)
    # };

    # float => {
    #   emit(:float_literal, data, token_array, ts, te)
    # };

    # assignment => {
    #   emit(:assignment_operator, data, token_array, ts, te)
    # };

    # boolean => {
    #   emit(:boolean, data, token_array, ts, te)
    # };

    # identifier => {
    #   emit(:identifier, data, token_array, ts, te)
    # };

    # string => {
    #   emit(:string, data, token_array, ts, te)
    # };

    # space|',';

  # );

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  integer     = ('+'|'-')?[0-9]+;
  float       = ('+'|'-')?[0-9]+'.'[0-9]+;
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+;
  string      = ["][^"]*["];
  boolean     = '.';
  id_kw       = 'ID'|'Number';

  action call_key { fcall key ; }
  action call_value {}

  n := ( identifier );

  key := |*
    'ID'|'Number'  => {
               emit(:identifier, data, token_array, ts, te)
             };
  *|;

  # parser
  main := ( ('<'|',') @call_key '=' @call_value )+ ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

  # key := (

    # integer => {
    #   emit(:integer_literal, data, token_array, ts, te)
    # };

    # float => {
    #   emit(:float_literal, data, token_array, ts, te)
    # };

    # assignment => {
    #   emit(:assignment_operator, data, token_array, ts, te)
    # };

    # boolean => {
    #   emit(:boolean, data, token_array, ts, te)
    # };

    # identifier => {
    #   emit(:identifier, data, token_array, ts, te)
    # };

    # string => {
    #   emit(:string, data, token_array, ts, te)
    # };

    # space|',';

  # );


In the port from flex to ragel, the Name separator :: in Ragel code was lost. Added it back.

/*
 *  Copyright 2006-2007 Adrian Thurston <thurston@cs.queensu.ca>
 */

/*  This file is part of Ragel.
 *
 *  Ragel is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Ragel is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Ragel; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "ragel.h"
#include "rlparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

extern char *Parser_lelNames[];

/* This is used for tracking the current stack of include file/machine pairs. It is
 * is used to detect and recursive include structure. */
struct IncludeStackItem
{
	IncludeStackItem( char *fileName, char *sectionName )
		: fileName(fileName), sectionName(sectionName) {}

	char *fileName;
	char *sectionName;
};

typedef Vector<IncludeStackItem> IncludeStack;

enum InlineBlockType
{
	CurlyDelimited,
	SemiTerminated
};

struct Scanner
{
	Scanner( char *fileName, istream &input, ostream &output,
			Parser *inclToParser, char *inclSectionTarg,
			int includeDepth )
	:
		fileName(fileName), input(input), output(output),
		inclToParser(inclToParser),
		inclSectionTarg(inclSectionTarg),
		includeDepth(includeDepth),
		line(1), column(1), lastnl(0),
		parser(0), ignoreSection(false),
		parserExistsError(false),
		whitespaceOn(true),
		lastToken(0)
		{}

	bool recursiveInclude( char *inclFileName, char *inclSectionName );

	char *prepareFileName( char *fileName, int len )
	{
		bool caseInsensitive;
		Token tokenFnStr, tokenRes;
		tokenFnStr.data = fileName;
		tokenFnStr.length = len;
		tokenFnStr.prepareLitString( tokenRes, caseInsensitive );
		return tokenRes.data;
	}

	void init();
	void token( int type, char *start, char *end );
	void token( int type, char c );
	void token( int type );
	void updateCol();
	void startSection();
	void endSection();
	void do_scan();
	bool active();
	ostream &scan_error();

	char *fileName;
	istream &input;
	ostream &output;
	Parser *inclToParser;
	char *inclSectionTarg;
	int includeDepth;

	int cs;
	int line;
	char *word, *lit;
	int word_len, lit_len;
	InputLoc sectionLoc;
	char *tokstart, *tokend;
	int column;
	char *lastnl;

	/* Set by machine statements, these persist from section to section
	 * allowing for unnamed sections. */
	Parser *parser;
	bool ignoreSection;
	IncludeStack includeStack;

	/* This is set if ragel has already emitted an error stating that
	 * no section name has been seen and thus no parser exists. */
	bool parserExistsError;

	/* This is for inline code. By default it is on. It goes off for
	 * statements and values in inline blocks which are parsed. */
	bool whitespaceOn;

	/* Keeps a record of the previous token sent to the section parser. */
	int lastToken;
};

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void Scanner::init( )
{
	%% write init;
}

bool Scanner::active()
{
	if ( ignoreSection )
		return false;

	if ( parser == 0 && ! parserExistsError ) {
		scan_error() << "there is no previous specification name" << endl;
		parserExistsError = true;
	}

	if ( parser == 0 )
		return false;

	return true;
}

ostream &Scanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool Scanner::recursiveInclude( char *inclFileName, char *inclSectionName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 &&
				strcmp( si->sectionName, inclSectionName ) == 0 )
		{
			return true;
		}
	}
	return false;
}

void Scanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = tokstart;
	//cerr << "adding " << tokend - from << " to column" << endl;
	column += tokend - from;
	lastnl = 0;
}

void Scanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void Scanner::token( int type )
{
	token( type, 0, 0 );
}

%%{
	machine section_parse;

	# This relies on the the kelbt implementation and the order
	# that tokens are declared.
	KW_Machine = 128;
	KW_Include = 129;
	KW_Write = 130;
	TK_Word = 131;
	TK_Literal = 132;

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_word { word = tokdata; word_len = toklen; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_machine
	{
		/* Assign a name to the machine. */
		char *machine = word;

		if ( inclSectionTarg == 0 ) {
			ignoreSection = false;

			ParserDictEl *pdEl = parserDict.find( machine );
			if ( pdEl == 0 ) {
				pdEl = new ParserDictEl( machine );
				pdEl->value = new Parser( fileName, machine, sectionLoc );
				pdEl->value->init();
				parserDict.insert( pdEl );
			}

			parser = pdEl->value;
		}
		else if ( strcmp( inclSectionTarg, machine ) == 0 ) {
			/* found include target */
			ignoreSection = false;
			parser = inclToParser;
		}
		else {
			/* ignoring section */
			ignoreSection = true;
			parser = 0;
		}
	}

	machine_stmt =
		( KW_Machine TK_Word @store_word ';' ) @handle_machine
		<>err mach_err <>eof mach_err;

	action handle_include
	{
		if ( active() ) {
			char *inclSectionName = word;
			char *inclFileName = 0;

			/* Implement defaults for the input file and section name. */
			if ( inclSectionName == 0 )
				inclSectionName = parser->sectionName;

			if ( lit != 0 )
				inclFileName = prepareFileName( lit, lit_len );
			else
				inclFileName = fileName;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( fileName, parser->sectionName ) );

			if ( recursiveInclude( inclFileName, inclSectionName ) )
				scan_error() << "include: this is a recursive include operation" << endl;
			else {
				/* Open the input file for reading. */
				ifstream *inFile = new ifstream( inclFileName );
				if ( ! inFile->is_open() ) {
					scan_error() << "include: could not open " <<
							inclFileName << " for reading" << endl;
				}

				Scanner scanner( inclFileName, *inFile, output, parser,
						inclSectionName, includeDepth+1 );
				scanner.init();
				scanner.do_scan( );
				delete inFile;
			}

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );
		}
	}

	include_names = (
		TK_Word @store_word ( TK_Literal @store_lit )? |
		TK_Literal @store_lit
	) >clear_words;

	include_stmt =
		( KW_Include include_names ';' ) @handle_include
		<>err incl_err <>eof incl_err;

	action write_command
	{
		if ( active() && machineSpec == 0 && machineName == 0 ) {
			output << "<write"
					" def_name=\"" << parser->sectionName << "\""
					" line=\"" << line << "\""
					" col=\"" << column << "\""
					">";
		}
	}

	action write_arg
	{
		if ( active() && machineSpec == 0 && machineName == 0 )
			output << "<arg>" << tokdata << "</arg>";
	}

	action write_close
	{
		if ( active() && machineSpec == 0 && machineName == 0 )
			output << "</write>\n";
	}

	write_stmt =
		( KW_Write @write_command
		( TK_Word @write_arg )+ ';' @write_close )
		<>err write_err <>eof write_err;

	action handle_token
	{
		/* Send the token off to the parser. */
		if ( active() ) {
			InputLoc loc;

			#if 0
			cerr << "scanner:" << line << ":" << column <<
					": sending token to the parser " << Parser_lelNames[*p];
			cerr << " " << toklen;
			if ( tokdata != 0 )
				cerr << " " << tokdata;
			cerr << endl;
			#endif

			loc.fileName = fileName;
			loc.line = line;
			loc.col = column;

			parser->token( loc, type, tokdata, toklen );
		}
	}

	# Catch everything else.
	everything_else = ^( KW_Machine | KW_Include | KW_Write ) @handle_token;

	main := (
		machine_stmt |
		include_stmt |
		write_stmt |
		everything_else
	)*;
}%%

void Scanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();

	/* Record the last token for use in controlling the scan of subsequent
	 * tokens. */
	lastToken = type;
}

void Scanner::startSection( )
{
	parserExistsError = false;

	if ( includeDepth == 0 ) {
		if ( machineSpec == 0 && machineName == 0 )
			output << "</host>\n";
	}

	sectionLoc.fileName = fileName;
	sectionLoc.line = line;
	sectionLoc.col = 0;
}

void Scanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	%%{
		machine section_parse;
		write eof;
	}%%

	/* Close off the section with the parser. */
	if ( active() ) {
		InputLoc loc;
		loc.fileName = fileName;
		loc.line = line;
		loc.col = 0;

		parser->token( loc, TK_EndSection, 0, 0 );
	}

	if ( includeDepth == 0 ) {
		if ( machineSpec == 0 && machineName == 0 ) {
			/* The end section may include a newline on the end, so
			 * we use the last line, which will count the newline. */
			output << "<host line=\"" << line << "\">";
		}
	}
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	c_comment =
		'/*' ( any | NL )* :>> '*/';

	cpp_comment =
		'//' [^\n]* NL;

	c_cpp_comment = c_comment | cpp_comment;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	# An inline block of code. This is specified as a scanned, but is sent to
	# the parser as one long block. The inline_block pointer is used to handle
	# the preservation of the data.
	inline_code := |*
		# Inline expression keywords.
		"fpc" => { token( KW_PChar ); };
		"fc" => { token( KW_Char ); };
		"fcurs" => { token( KW_CurState ); };
		"ftargs" => { token( KW_TargState ); };
		"fentry" => {
			whitespaceOn = false;
			token( KW_Entry );
		};

		# Inline statement keywords.
		"fhold" => {
			whitespaceOn = false;
			token( KW_Hold );
		};
		"fexec" => { token( KW_Exec, 0, 0 ); };
		"fgoto" => {
			whitespaceOn = false;
			token( KW_Goto );
		};
		"fnext" => {
			whitespaceOn = false;
			token( KW_Next );
		};
		"fcall" => {
			whitespaceOn = false;
			token( KW_Call );
		};
		"fret" => {
			whitespaceOn = false;
			token( KW_Ret );
		};
		"fbreak" => {
			whitespaceOn = false;
			token( KW_Break );
		};

		ident => { token( TK_Word, tokstart, tokend ); };

		number => { token( TK_UInt, tokstart, tokend ); };
		hex_number => { token( TK_Hex, tokstart, tokend ); };

		( s_literal | d_literal )
			=> { token( IL_Literal, tokstart, tokend ); };

		whitespace+ => {
			if ( whitespaceOn )
				token( IL_WhiteSpace, tokstart, tokend );
		};
		c_cpp_comment => { token( IL_Comment, tokstart, tokend ); };

		"::" => { token( TK_NameSep, tokstart, tokend ); };

		# Some symbols need to go to the parser as with their cardinal value as
		# the token type (as opposed to being sent as anonymous symbols)
		# because they are part of the sequences which we interpret. The * ) ;
		# symbols cause whitespace parsing to come back on. This gets turned
		# off by some keywords.

		";" => {
			whitespaceOn = true;
			token( *tokstart, tokstart, tokend );
			if ( inlineBlockType == SemiTerminated )
				fgoto parser_def;
		};

		[*)] => {
			whitespaceOn = true;
			token( *tokstart, tokstart, tokend );
		};

		[,(] => { token( *tokstart, tokstart, tokend ); };

		'{' => {
			token( IL_Symbol, tokstart, tokend );
			curly_count += 1;
		};

		'}' => {
			if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
				/* Inline code block ends. */
				token( '}' );
				fgoto parser_def;
			}
			else {
				/* Either a semi terminated inline block or only the closing
				 * brace of some inner scope, not the block's closing brace. */
				token( IL_Symbol, tokstart, tokend );
			}
		};

		EOF => {
			scan_error() << "unterminated code block" << endl;
		};

		# Send every other character as a symbol.
		any => { token( IL_Symbol, tokstart, tokend ); };
	*|;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( RE_Char, '\0' ); };
		'\\a' => { token( RE_Char, '\a' ); };
		'\\b' => { token( RE_Char, '\b' ); };
		'\\t' => { token( RE_Char, '\t' ); };
		'\\n' => { token( RE_Char, '\n' ); };
		'\\v' => { token( RE_Char, '\v' ); };
		'\\f' => { token( RE_Char, '\f' ); };
		'\\r' => { token( RE_Char, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( RE_Char, tokstart+1, tokend ); };

		# Range dash in an OR expression.
		'-' => { token( RE_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( RE_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( RE_Char, tokstart, tokend ); };

	*|;

	re_literal := |*
		# Escape sequences in regular expressions.
		'\\0' => { token( RE_Char, '\0' ); };
		'\\a' => { token( RE_Char, '\a' ); };
		'\\b' => { token( RE_Char, '\b' ); };
		'\\t' => { token( RE_Char, '\t' ); };
		'\\n' => { token( RE_Char, '\n' ); };
		'\\v' => { token( RE_Char, '\v' ); };
		'\\f' => { token( RE_Char, '\f' ); };
		'\\r' => { token( RE_Char, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( RE_Char, tokstart+1, tokend ); };

		# Terminate an OR expression.
		'/' [i]? => {
			token( RE_Slash, tokstart, tokend );
			fgoto parser_def;
		};

		# Special characters.
		'.' => { token( RE_Dot ); };
		'*' => { token( RE_Star ); };

		'[' => { token( RE_SqOpen ); fcall or_literal; };
		'[^' => { token( RE_SqOpenNeg ); fcall or_literal; };

		EOF => {
			scan_error() << "unterminated regular expression" << endl;
		};

		# Characters in an OR expression.
		[^\/] => { token( RE_Char, tokstart, tokend ); };
	*|;

	# We need a separate token space here to avoid the ragel keywords.
	write_statement := |*
		ident => { token( TK_Word, tokstart, tokend ); } ;
		[ \t\n]+ => { updateCol(); };
		';' => { token( ';' ); fgoto parser_def; };

		EOF => {
			scan_error() << "unterminated write statement" << endl;
		};
	*|;

	# Parser definitions.
	parser_def := |*
		'machine' => { token( KW_Machine ); };
		'include' => { token( KW_Include ); };
		'write' => {
			token( KW_Write );
			fgoto write_statement;
		};
		'action' => { token( KW_Action ); };
		'alphtype' => { token( KW_AlphType ); };

		# FIXME: Enable this post 5.17.
		# 'range' => { token( KW_Range ); };

		'getkey' => {
			token( KW_GetKey );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'access' => {
			token( KW_Access );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'variable' => {
			token( KW_Variable );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'when' => { token( KW_When ); };
		'eof' => { token( KW_Eof ); };
		'err' => { token( KW_Err ); };
		'lerr' => { token( KW_Lerr ); };
		'to' => { token( KW_To ); };
		'from' => { token( KW_From ); };
		'export' => { token( KW_Export ); };
		'entry' => { token( KW_Entry ); };

		# Identifiers.
		ident => { token( TK_Word, tokstart, tokend ); } ;

		# Numbers
		number => { token( TK_UInt, tokstart, tokend ); };
		hex_number => { token( TK_Hex, tokstart, tokend ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, tokstart, tokend ); };

		'[' => { token( RE_SqOpen ); fcall or_literal; };
		'[^' => { token( RE_SqOpenNeg ); fcall or_literal; };

		'/' => { token( RE_Slash ); fgoto re_literal; };

		# Ignore.
		pound_comment => { updateCol(); };

		':=' => { token( TK_ColonEquals ); };

		# To State Actions.
		">~" => { token( TK_StartToState ); };
		"$~" => { token( TK_AllToState ); };
		"%~" => { token( TK_FinalToState ); };
		"<~" => { token( TK_NotStartToState ); };
		"@~" => { token( TK_NotFinalToState ); };
		"<>~" => { token( TK_MiddleToState ); };

		# From State actions
		">*" => { token( TK_StartFromState ); };
		"$*" => { token( TK_AllFromState ); };
		"%*" => { token( TK_FinalFromState ); };
		"<*" => { token( TK_NotStartFromState ); };
		"@*" => { token( TK_NotFinalFromState ); };
		"<>*" => { token( TK_MiddleFromState ); };

		# EOF Actions.
		">/" => { token( TK_StartEOF ); };
		"$/" => { token( TK_AllEOF ); };
		"%/" => { token( TK_FinalEOF ); };
		"</" => { token( TK_NotStartEOF ); };
		"@/" => { token( TK_NotFinalEOF ); };
		"<>/" => { token( TK_MiddleEOF ); };

		# Global Error actions.
		">!" => { token( TK_StartGblError ); };
		"$!" => { token( TK_AllGblError ); };
		"%!" => { token( TK_FinalGblError ); };
		"<!" => { token( TK_NotStartGblError ); };
		"@!" => { token( TK_NotFinalGblError ); };
		"<>!" => { token( TK_MiddleGblError ); };

		# Local error actions.
		">^" => { token( TK_StartLocalError ); };
		"$^" => { token( TK_AllLocalError ); };
		"%^" => { token( TK_FinalLocalError ); };
		"<^" => { token( TK_NotStartLocalError ); };
		"@^" => { token( TK_NotFinalLocalError ); };
		"<>^" => { token( TK_MiddleLocalError ); };

		# Middle.
		"<>" => { token( TK_Middle ); };

		# Conditions.
		'>?' => { token( TK_StartCond ); };
		'$?' => { token( TK_AllCond ); };
		'%?' => { token( TK_LeavingCond ); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };
		'->' => { token( TK_Arrow ); };
		'=>' => { token( TK_DoubleArrow ); };

		":>"  => { token( TK_ColonGt ); };
		":>>" => { token( TK_ColonGtGt ); };
		"<:"  => { token( TK_LtColon ); };

		# Opening of longest match.
		"|*" => { token( TK_BarStar ); };

		# Separater for name references.
		"::" => { token( TK_NameSep, tokstart, tokend ); };

		'}%%' => {
			updateCol();
			endSection();
			fgoto main;
		};

		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => {
			updateCol();
			if ( singleLineSpec ) {
				endSection();
				fgoto main;
			}
		};

		'{' => {
			if ( lastToken == KW_Export || lastToken == KW_Entry )
				token( '{' );
			else {
				token( '{' );
				curly_count = 1;
				inlineBlockType = CurlyDelimited;
				fgoto inline_code;
			}
		};

		EOF => {
			scan_error() << "unterminated ragel section" << endl;
		};

		any => { token( *tokstart ); } ;
	*|;

	action pass {
		updateCol();

		/* If no errors and we are at the bottom of the include stack (the
		 * source file listed on the command line) then write out the data. */
		if ( includeDepth == 0 && machineSpec == 0 && machineName == 0 )
			xmlEscapeHost( output, tokstart, tokend-tokstart );
	}

	# Outside code scanner. These tokens get passed through.
	main := |*
		ident => pass;
		number => pass;
		c_cpp_comment => pass;
		s_literal | d_literal => pass;
		'%%{' => {
			updateCol();
			singleLineSpec = false;
			startSection();
			fgoto parser_def;
		};
		'%%' => {
			updateCol();
			singleLineSpec = true;
			startSection();
			fgoto parser_def;
		};
		whitespace+ => pass;
		EOF;
		any => pass;
	*|;

}%%

%% write data;

void Scanner::do_scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[1];
	int curly_count = 0;
	bool execute = true;
	bool singleLineSpec = false;
	InlineBlockType inlineBlockType;

	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( tokstart != 0 )
				tokstart = newbuf + ( tokstart - buf );
			tokend = newbuf + ( tokend - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "scanner error" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = tokstart;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( tokstart != 0 )
				tokstart -= shiftback;
			tokend -= shiftback;

			preserve = buf;
		}
	}

	delete[] buf;
}

void scan( char *fileName, istream &input, ostream &output )
{
	Scanner scanner( fileName, input, output, 0, 0, 0 );
	scanner.init();
	scanner.do_scan();

	InputLoc eofLoc;
	eofLoc.fileName = fileName;
	eofLoc.col = 1;
	eofLoc.line = scanner.line;
}

/*
 *  Copyright 2006-2007 Adrian Thurston <thurston@cs.queensu.ca>
 */

/*  This file is part of Ragel.
 *
 *  Ragel is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Ragel is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Ragel; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "ragel.h"
#include "rlparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

extern char *Parser_lelNames[];

/* This is used for tracking the current stack of include file/machine pairs. It is
 * is used to detect and recursive include structure. */
struct IncludeStackItem
{
	IncludeStackItem( char *fileName, char *sectionName )
		: fileName(fileName), sectionName(sectionName) {}

	char *fileName;
	char *sectionName;
};

typedef Vector<IncludeStackItem> IncludeStack;

enum InlineBlockType
{
	CurlyDelimited,
	SemiTerminated
};

struct Scanner
{
	Scanner( char *fileName, istream &input, ostream &output,
			Parser *inclToParser, char *inclSectionTarg,
			int includeDepth )
	:
		fileName(fileName), input(input), output(output),
		inclToParser(inclToParser),
		inclSectionTarg(inclSectionTarg),
		includeDepth(includeDepth),
		line(1), column(1), lastnl(0),
		parser(0), ignoreSection(false),
		parserExistsError(false),
		whitespaceOn(true),
		lastToken(0)
		{}

	bool recursiveInclude( char *inclFileName, char *inclSectionName );

	char *prepareFileName( char *fileName, int len )
	{
		bool caseInsensitive;
		Token tokenFnStr, tokenRes;
		tokenFnStr.data = fileName;
		tokenFnStr.length = len;
		tokenFnStr.prepareLitString( tokenRes, caseInsensitive );
		return tokenRes.data;
	}

	void init();
	void token( int type, char *start, char *end );
	void token( int type, char c );
	void token( int type );
	void updateCol();
	void startSection();
	void endSection();
	void do_scan();
	bool active();
	ostream &scan_error();

	char *fileName;
	istream &input;
	ostream &output;
	Parser *inclToParser;
	char *inclSectionTarg;
	int includeDepth;

	int cs;
	int line;
	char *word, *lit;
	int word_len, lit_len;
	InputLoc sectionLoc;
	char *tokstart, *tokend;
	int column;
	char *lastnl;

	/* Set by machine statements, these persist from section to section
	 * allowing for unnamed sections. */
	Parser *parser;
	bool ignoreSection;
	IncludeStack includeStack;

	/* This is set if ragel has already emitted an error stating that
	 * no section name has been seen and thus no parser exists. */
	bool parserExistsError;

	/* This is for inline code. By default it is on. It goes off for
	 * statements and values in inline blocks which are parsed. */
	bool whitespaceOn;

	/* Keeps a record of the previous token sent to the section parser. */
	int lastToken;
};

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void Scanner::init( )
{
	%% write init;
}

bool Scanner::active()
{
	if ( ignoreSection )
		return false;

	if ( parser == 0 && ! parserExistsError ) {
		scan_error() << "there is no previous specification name" << endl;
		parserExistsError = true;
	}

	if ( parser == 0 )
		return false;

	return true;
}

ostream &Scanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool Scanner::recursiveInclude( char *inclFileName, char *inclSectionName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 &&
				strcmp( si->sectionName, inclSectionName ) == 0 )
		{
			return true;
		}
	}
	return false;
}

void Scanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = tokstart;
	//cerr << "adding " << tokend - from << " to column" << endl;
	column += tokend - from;
	lastnl = 0;
}

void Scanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void Scanner::token( int type )
{
	token( type, 0, 0 );
}

%%{
	machine section_parse;

	# This relies on the the kelbt implementation and the order
	# that tokens are declared.
	KW_Machine = 128;
	KW_Include = 129;
	KW_Write = 130;
	TK_Word = 131;
	TK_Literal = 132;

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_word { word = tokdata; word_len = toklen; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_machine
	{
		/* Assign a name to the machine. */
		char *machine = word;

		if ( inclSectionTarg == 0 ) {
			ignoreSection = false;

			ParserDictEl *pdEl = parserDict.find( machine );
			if ( pdEl == 0 ) {
				pdEl = new ParserDictEl( machine );
				pdEl->value = new Parser( fileName, machine, sectionLoc );
				pdEl->value->init();
				parserDict.insert( pdEl );
			}

			parser = pdEl->value;
		}
		else if ( strcmp( inclSectionTarg, machine ) == 0 ) {
			/* found include target */
			ignoreSection = false;
			parser = inclToParser;
		}
		else {
			/* ignoring section */
			ignoreSection = true;
			parser = 0;
		}
	}

	machine_stmt =
		( KW_Machine TK_Word @store_word ';' ) @handle_machine
		<>err mach_err <>eof mach_err;

	action handle_include
	{
		if ( active() ) {
			char *inclSectionName = word;
			char *inclFileName = 0;

			/* Implement defaults for the input file and section name. */
			if ( inclSectionName == 0 )
				inclSectionName = parser->sectionName;

			if ( lit != 0 )
				inclFileName = prepareFileName( lit, lit_len );
			else
				inclFileName = fileName;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( fileName, parser->sectionName ) );

			if ( recursiveInclude( inclFileName, inclSectionName ) )
				scan_error() << "include: this is a recursive include operation" << endl;
			else {
				/* Open the input file for reading. */
				ifstream *inFile = new ifstream( inclFileName );
				if ( ! inFile->is_open() ) {
					scan_error() << "include: could not open " <<
							inclFileName << " for reading" << endl;
				}

				Scanner scanner( inclFileName, *inFile, output, parser,
						inclSectionName, includeDepth+1 );
				scanner.init();
				scanner.do_scan( );
				delete inFile;
			}

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );
		}
	}

	include_names = (
		TK_Word @store_word ( TK_Literal @store_lit )? |
		TK_Literal @store_lit
	) >clear_words;

	include_stmt =
		( KW_Include include_names ';' ) @handle_include
		<>err incl_err <>eof incl_err;

	action write_command
	{
		if ( active() && machineSpec == 0 && machineName == 0 ) {
			output << "<write"
					" def_name=\"" << parser->sectionName << "\""
					" line=\"" << line << "\""
					" col=\"" << column << "\""
					">";
		}
	}

	action write_arg
	{
		if ( active() && machineSpec == 0 && machineName == 0 )
			output << "<arg>" << tokdata << "</arg>";
	}

	action write_close
	{
		if ( active() && machineSpec == 0 && machineName == 0 )
			output << "</write>\n";
	}

	write_stmt =
		( KW_Write @write_command
		( TK_Word @write_arg )+ ';' @write_close )
		<>err write_err <>eof write_err;

	action handle_token
	{
		/* Send the token off to the parser. */
		if ( active() ) {
			InputLoc loc;

			#if 0
			cerr << "scanner:" << line << ":" << column <<
					": sending token to the parser " << Parser_lelNames[*p];
			cerr << " " << toklen;
			if ( tokdata != 0 )
				cerr << " " << tokdata;
			cerr << endl;
			#endif

			loc.fileName = fileName;
			loc.line = line;
			loc.col = column;

			parser->token( loc, type, tokdata, toklen );
		}
	}

	# Catch everything else.
	everything_else = ^( KW_Machine | KW_Include | KW_Write ) @handle_token;

	main := (
		machine_stmt |
		include_stmt |
		write_stmt |
		everything_else
	)*;
}%%

void Scanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();

	/* Record the last token for use in controlling the scan of subsequent
	 * tokens. */
	lastToken = type;
}

void Scanner::startSection( )
{
	parserExistsError = false;

	if ( includeDepth == 0 ) {
		if ( machineSpec == 0 && machineName == 0 )
			output << "</host>\n";
	}

	sectionLoc.fileName = fileName;
	sectionLoc.line = line;
	sectionLoc.col = 0;
}

void Scanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	%%{
		machine section_parse;
		write eof;
	}%%

	/* Close off the section with the parser. */
	if ( active() ) {
		InputLoc loc;
		loc.fileName = fileName;
		loc.line = line;
		loc.col = 0;

		parser->token( loc, TK_EndSection, 0, 0 );
	}

	if ( includeDepth == 0 ) {
		if ( machineSpec == 0 && machineName == 0 ) {
			/* The end section may include a newline on the end, so
			 * we use the last line, which will count the newline. */
			output << "<host line=\"" << line << "\">";
		}
	}
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	c_comment =
		'/*' ( any | NL )* :>> '*/';

	cpp_comment =
		'//' [^\n]* NL;

	c_cpp_comment = c_comment | cpp_comment;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	# An inline block of code. This is specified as a scanned, but is sent to
	# the parser as one long block. The inline_block pointer is used to handle
	# the preservation of the data.
	inline_code := |*
		# Inline expression keywords.
		"fpc" => { token( KW_PChar ); };
		"fc" => { token( KW_Char ); };
		"fcurs" => { token( KW_CurState ); };
		"ftargs" => { token( KW_TargState ); };
		"fentry" => {
			whitespaceOn = false;
			token( KW_Entry );
		};

		# Inline statement keywords.
		"fhold" => {
			whitespaceOn = false;
			token( KW_Hold );
		};
		"fexec" => { token( KW_Exec, 0, 0 ); };
		"fgoto" => {
			whitespaceOn = false;
			token( KW_Goto );
		};
		"fnext" => {
			whitespaceOn = false;
			token( KW_Next );
		};
		"fcall" => {
			whitespaceOn = false;
			token( KW_Call );
		};
		"fret" => {
			whitespaceOn = false;
			token( KW_Ret );
		};
		"fbreak" => {
			whitespaceOn = false;
			token( KW_Break );
		};

		ident => { token( TK_Word, tokstart, tokend ); };

		number => { token( TK_UInt, tokstart, tokend ); };
		hex_number => { token( TK_Hex, tokstart, tokend ); };

		( s_literal | d_literal )
			=> { token( IL_Literal, tokstart, tokend ); };

		whitespace+ => {
			if ( whitespaceOn )
				token( IL_WhiteSpace, tokstart, tokend );
		};
		c_cpp_comment => { token( IL_Comment, tokstart, tokend ); };

		"::" => { token( TK_NameSep, tokstart, tokend ); };

		# Some symbols need to go to the parser as with their cardinal value as
		# the token type (as opposed to being sent as anonymous symbols)
		# because they are part of the sequences which we interpret. The * ) ;
		# symbols cause whitespace parsing to come back on. This gets turned
		# off by some keywords.

		";" => {
			whitespaceOn = true;
			token( *tokstart, tokstart, tokend );
			if ( inlineBlockType == SemiTerminated )
				fgoto parser_def;
		};

		[*)] => {
			whitespaceOn = true;
			token( *tokstart, tokstart, tokend );
		};

		[,(] => { token( *tokstart, tokstart, tokend ); };

		'{' => {
			token( IL_Symbol, tokstart, tokend );
			curly_count += 1;
		};

		'}' => {
			if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
				/* Inline code block ends. */
				token( '}' );
				fgoto parser_def;
			}
			else {
				/* Either a semi terminated inline block or only the closing
				 * brace of some inner scope, not the block's closing brace. */
				token( IL_Symbol, tokstart, tokend );
			}
		};

		EOF => {
			scan_error() << "unterminated code block" << endl;
		};

		# Send every other character as a symbol.
		any => { token( IL_Symbol, tokstart, tokend ); };
	*|;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( RE_Char, '\0' ); };
		'\\a' => { token( RE_Char, '\a' ); };
		'\\b' => { token( RE_Char, '\b' ); };
		'\\t' => { token( RE_Char, '\t' ); };
		'\\n' => { token( RE_Char, '\n' ); };
		'\\v' => { token( RE_Char, '\v' ); };
		'\\f' => { token( RE_Char, '\f' ); };
		'\\r' => { token( RE_Char, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( RE_Char, tokstart+1, tokend ); };

		# Range dash in an OR expression.
		'-' => { token( RE_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( RE_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( RE_Char, tokstart, tokend ); };

	*|;

	re_literal := |*
		# Escape sequences in regular expressions.
		'\\0' => { token( RE_Char, '\0' ); };
		'\\a' => { token( RE_Char, '\a' ); };
		'\\b' => { token( RE_Char, '\b' ); };
		'\\t' => { token( RE_Char, '\t' ); };
		'\\n' => { token( RE_Char, '\n' ); };
		'\\v' => { token( RE_Char, '\v' ); };
		'\\f' => { token( RE_Char, '\f' ); };
		'\\r' => { token( RE_Char, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( RE_Char, tokstart+1, tokend ); };

		# Terminate an OR expression.
		'/' [i]? => {
			token( RE_Slash, tokstart, tokend );
			fgoto parser_def;
		};

		# Special characters.
		'.' => { token( RE_Dot ); };
		'*' => { token( RE_Star ); };

		'[' => { token( RE_SqOpen ); fcall or_literal; };
		'[^' => { token( RE_SqOpenNeg ); fcall or_literal; };

		EOF => {
			scan_error() << "unterminated regular expression" << endl;
		};

		# Characters in an OR expression.
		[^\/] => { token( RE_Char, tokstart, tokend ); };
	*|;

	# We need a separate token space here to avoid the ragel keywords.
	write_statement := |*
		ident => { token( TK_Word, tokstart, tokend ); } ;
		[ \t\n]+ => { updateCol(); };
		';' => { token( ';' ); fgoto parser_def; };

		EOF => {
			scan_error() << "unterminated write statement" << endl;
		};
	*|;

	# Parser definitions.
	parser_def := |*
		'machine' => { token( KW_Machine ); };
		'include' => { token( KW_Include ); };
		'write' => {
			token( KW_Write );
			fgoto write_statement;
		};
		'action' => { token( KW_Action ); };
		'alphtype' => { token( KW_AlphType ); };

		# FIXME: Enable this post 5.17.
		# 'range' => { token( KW_Range ); };

		'getkey' => {
			token( KW_GetKey );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'access' => {
			token( KW_Access );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'variable' => {
			token( KW_Variable );
			inlineBlockType = SemiTerminated;
			fgoto inline_code;
		};
		'when' => { token( KW_When ); };
		'eof' => { token( KW_Eof ); };
		'err' => { token( KW_Err ); };
		'lerr' => { token( KW_Lerr ); };
		'to' => { token( KW_To ); };
		'from' => { token( KW_From ); };
		'export' => { token( KW_Export ); };
		'entry' => { token( KW_Entry ); };

		# Identifiers.
		ident => { token( TK_Word, tokstart, tokend ); } ;

		# Numbers
		number => { token( TK_UInt, tokstart, tokend ); };
		hex_number => { token( TK_Hex, tokstart, tokend ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, tokstart, tokend ); };

		'[' => { token( RE_SqOpen ); fcall or_literal; };
		'[^' => { token( RE_SqOpenNeg ); fcall or_literal; };

		'/' => { token( RE_Slash ); fgoto re_literal; };

		# Ignore.
		pound_comment => { updateCol(); };

		':=' => { token( TK_ColonEquals ); };

		# To State Actions.
		">~" => { token( TK_StartToState ); };
		"$~" => { token( TK_AllToState ); };
		"%~" => { token( TK_FinalToState ); };
		"<~" => { token( TK_NotStartToState ); };
		"@~" => { token( TK_NotFinalToState ); };
		"<>~" => { token( TK_MiddleToState ); };

		# From State actions
		">*" => { token( TK_StartFromState ); };
		"$*" => { token( TK_AllFromState ); };
		"%*" => { token( TK_FinalFromState ); };
		"<*" => { token( TK_NotStartFromState ); };
		"@*" => { token( TK_NotFinalFromState ); };
		"<>*" => { token( TK_MiddleFromState ); };

		# EOF Actions.
		">/" => { token( TK_StartEOF ); };
		"$/" => { token( TK_AllEOF ); };
		"%/" => { token( TK_FinalEOF ); };
		"</" => { token( TK_NotStartEOF ); };
		"@/" => { token( TK_NotFinalEOF ); };
		"<>/" => { token( TK_MiddleEOF ); };

		# Global Error actions.
		">!" => { token( TK_StartGblError ); };
		"$!" => { token( TK_AllGblError ); };
		"%!" => { token( TK_FinalGblError ); };
		"<!" => { token( TK_NotStartGblError ); };
		"@!" => { token( TK_NotFinalGblError ); };
		"<>!" => { token( TK_MiddleGblError ); };

		# Local error actions.
		">^" => { token( TK_StartLocalError ); };
		"$^" => { token( TK_AllLocalError ); };
		"%^" => { token( TK_FinalLocalError ); };
		"<^" => { token( TK_NotStartLocalError ); };
		"@^" => { token( TK_NotFinalLocalError ); };
		"<>^" => { token( TK_MiddleLocalError ); };

		# Middle.
		"<>" => { token( TK_Middle ); };

		# Conditions.
		'>?' => { token( TK_StartCond ); };
		'$?' => { token( TK_AllCond ); };
		'%?' => { token( TK_LeavingCond ); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };
		'->' => { token( TK_Arrow ); };
		'=>' => { token( TK_DoubleArrow ); };

		":>"  => { token( TK_ColonGt ); };
		":>>" => { token( TK_ColonGtGt ); };
		"<:"  => { token( TK_LtColon ); };

		# Opening of longest match.
		"|*" => { token( TK_BarStar ); };

		'}%%' => {
			updateCol();
			endSection();
			fgoto main;
		};

		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => {
			updateCol();
			if ( singleLineSpec ) {
				endSection();
				fgoto main;
			}
		};

		'{' => {
			if ( lastToken == KW_Export || lastToken == KW_Entry )
				token( '{' );
			else {
				token( '{' );
				curly_count = 1;
				inlineBlockType = CurlyDelimited;
				fgoto inline_code;
			}
		};

		EOF => {
			scan_error() << "unterminated ragel section" << endl;
		};

		any => { token( *tokstart ); } ;
	*|;

	action pass {
		updateCol();

		/* If no errors and we are at the bottom of the include stack (the
		 * source file listed on the command line) then write out the data. */
		if ( includeDepth == 0 && machineSpec == 0 && machineName == 0 )
			xmlEscapeHost( output, tokstart, tokend-tokstart );
	}

	# Outside code scanner. These tokens get passed through.
	main := |*
		ident => pass;
		number => pass;
		c_cpp_comment => pass;
		s_literal | d_literal => pass;
		'%%{' => {
			updateCol();
			singleLineSpec = false;
			startSection();
			fgoto parser_def;
		};
		'%%' => {
			updateCol();
			singleLineSpec = true;
			startSection();
			fgoto parser_def;
		};
		whitespace+ => pass;
		EOF;
		any => pass;
	*|;

}%%

%% write data;

void Scanner::do_scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[1];
	int curly_count = 0;
	bool execute = true;
	bool singleLineSpec = false;
	InlineBlockType inlineBlockType;

	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( tokstart != 0 )
				tokstart = newbuf + ( tokstart - buf );
			tokend = newbuf + ( tokend - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "scanner error" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = tokstart;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( tokstart != 0 )
				tokstart -= shiftback;
			tokend -= shiftback;

			preserve = buf;
		}
	}

	delete[] buf;
}

void scan( char *fileName, istream &input, ostream &output )
{
	Scanner scanner( fileName, input, output, 0, 0, 0 );
	scanner.init();
	scanner.do_scan();

	InputLoc eofLoc;
	eofLoc.fileName = fileName;
	eofLoc.col = 1;
	eofLoc.line = scanner.line;
}


Unbreak sctp stream parsing, now sctp works with mplayer-nemesi

/* -*- c -*- */

#include <stdbool.h>

#include "network/rtsp.h"
#include "network/rtp.h"
#include "network/ragel_parsers.h"

%% machine ragel_transport_header;

gboolean ragel_parse_transport_header(RTSP_Client *rtsp,
                                      RTP_transport *rtp_t,
                                      const char *header) {
    struct ParsedTransport transport;
    int cs;
    const char *p = header, *pe = p + strlen(p) +1;
    uint32_t portval; uint16_t chanval;

    %%{
        action start_port {
            portval = 0;
        }

        action count_port {
            portval = (portval*10) + (fc - '0');
        }

        action check_port {
            if ( portval > G_MAXUINT16 )
                portval = G_MAXUINT16;
        }

        action start_channel {
            chanval = 0;
        }

        action count_channel {
            chanval = (chanval*10) + (fc - '0');
        }

        action check_channel {
            if ( chanval > G_MAXUINT8 )
                chanval = G_MAXUINT8;
        }

        Port = digit{,5} >start_port @count_port %check_port;
        Channel = digit{,3} >start_channel @count_channel %check_channel;

        Unicast = ";unicast" %{transport.mode = TransportUnicast;};
        Multicast = ";multicast" %{transport.mode = TransportMulticast;};

        TransportParamName = ( alnum | '_' )+;
        TransportParamValue = (print - ('=' | ';' ))+;

        TransportParam = ";" . TransportParamName . ( '=' . TransportParamValue )?;

        ClientPort = ";client_port=" .
            Port%{transport.parameters.UDP.Unicast.port_rtp = portval;} .
            ( "-" . Port%{transport.parameters.UDP.Unicast.port_rtcp = portval;} );

        UnicastUDPParams = Unicast . ( ClientPort | TransportParam )+;
        MulticastUDPParams = Multicast . TransportParam+;

        UDPParams = ( UnicastUDPParams | MulticastUDPParams );

        Interleaved = ";interleaved=" .
            Channel%{transport.parameters.TCP.ich_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.TCP.ich_rtcp = chanval;} );

        TCPParams = ( Interleaved | TransportParam)+;

        Streams = ";streams=" .
            Channel%{transport.parameters.SCTP.ch_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.SCTP.ch_rtcp = chanval;} );

        SCTPParams = ( Streams | TransportParam)+;

        TransportUDP = ("/UDP")? %{transport.protocol = TransportUDP; }
            . UDPParams;
        TransportTCP = "/TCP" %{transport.protocol = TransportTCP; }
            . TCPParams;
        TransportSCTP = "/SCTP" %{transport.protocol = TransportSCTP; }
            . SCTPParams;

        action check_transport {
            /* If the transport is valid, our work is done */
            if ( check_parsed_transport(rtsp, rtp_t, &transport) )
                return true;
        }

        TransportSpec = "RTP/AVP" . ( TransportUDP|TransportTCP|TransportSCTP )
            > { memset(&transport, 0, sizeof(transport)); }
            % check_transport;

        main := TransportSpec . ( ',' . TransportSpec )* . 0;

        write data;
        write init;
        write exec;
    }%%

    return false;
}

/* -*- c -*- */

#include <stdbool.h>

#include "network/rtsp.h"
#include "network/rtp.h"
#include "network/ragel_parsers.h"

%% machine ragel_transport_header;

gboolean ragel_parse_transport_header(RTSP_Client *rtsp,
                                      RTP_transport *rtp_t,
                                      const char *header) {
    struct ParsedTransport transport;
    int cs;
    const char *p = header, *pe = p + strlen(p) +1;
    uint32_t portval; uint16_t chanval;

    %%{
        action start_port {
            portval = 0;
        }

        action count_port {
            portval = (portval*10) + (fc - '0');
        }

        action check_port {
            if ( portval > G_MAXUINT16 )
                portval = G_MAXUINT16;
        }

        action start_channel {
            chanval = 0;
        }

        action count_channel {
            chanval = (chanval*10) + (fc - '0');
        }

        action check_channel {
            if ( chanval > G_MAXUINT8 )
                chanval = G_MAXUINT8;
        }

        Port = digit{,5} >start_port @count_port %check_port;
        Channel = digit{,3} >start_channel @count_channel %check_channel;

        Unicast = ";unicast" %{transport.mode = TransportUnicast;};
        Multicast = ";multicast" %{transport.mode = TransportMulticast;};

        TransportParamName = ( alnum | '_' )+;
        TransportParamValue = (print - ('=' | ';' ))+;

        TransportParam = ";" . TransportParamName . ( '=' . TransportParamValue )?;

        ClientPort = ";client_port=" .
            Port%{transport.parameters.UDP.Unicast.port_rtp = portval;} .
            ( "-" . Port%{transport.parameters.UDP.Unicast.port_rtcp = portval;} );

        UnicastUDPParams = Unicast . ( ClientPort | TransportParam )+;
        MulticastUDPParams = Multicast . TransportParam+;

        UDPParams = ( UnicastUDPParams | MulticastUDPParams );

        Interleaved = ";interleaved=" .
            Channel%{transport.parameters.TCP.ich_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.TCP.ich_rtcp = chanval;} );

        TCPParams = ( Interleaved | TransportParam)+;

        Streams = ";streams" .
            Channel%{transport.parameters.SCTP.ch_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.SCTP.ch_rtcp = chanval;} );

        SCTPParams = TransportParam+;

        TransportUDP = ("/UDP")? %{transport.protocol = TransportUDP; }
            . UDPParams;
        TransportTCP = "/TCP" %{transport.protocol = TransportTCP; }
            . TCPParams;
        TransportSCTP = "/SCTP" %{transport.protocol = TransportSCTP; }
            . SCTPParams;

        action check_transport {
            /* If the transport is valid, our work is done */
            if ( check_parsed_transport(rtsp, rtp_t, &transport) )
                return true;
        }

        TransportSpec = "RTP/AVP" . ( TransportUDP|TransportTCP|TransportSCTP )
            > { memset(&transport, 0, sizeof(transport)); }
            % check_transport;

        main := TransportSpec . ( ',' . TransportSpec )* . 0;

        write data;
        write init;
        write exec;
    }%%

    return false;
}


updated expected output in rlscan test since adding more input


remove some debugging code

remove some debugging code

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

static VALUE eTnetsParserError;

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    if (pe - str > size) {
      TNETS_ERROR("Oops I almost accidentally the buffer.");
    }
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    while(sub_size > 0) {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    }

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    TNETS_T arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    while (sub_size > 0) {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);
    }
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    TNETS_ERROR("parse error"); // TODO: more helpful messages
  }

  # kind of a hack, but represents the end of the chunk.
  end = (any @error)?;

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' (('-' @neg)? (digit+ $collect_num)) %/wrap_num <: end;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = ('~' %/wrap_null) end;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.  Also add one for the colon.
  char *pe = p + ( (size < 11) ? size : 11 );
  char *eof = pe;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE mTnetstrings = rb_define_module("TNETS");

  rb_define_method(mTnetstrings, "c_parse", rb_parse_tnets, 1);
  eTnetsParserError = rb_define_class_under(mTnetstrings, "ParserError", rb_eIOError);
}

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

static VALUE eTnetsParserError;

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    if (pe - str > size) {
      TNETS_ERROR("Oops I almost accidentally the buffer.");
    }
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    while(sub_size > 0) {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    }

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    TNETS_T arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    while (sub_size > 0) {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);
    }
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n");
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
    TNETS_ERROR("parse error"); // TODO: more helpful messages
  }

  # kind of a hack, but represents the end of the chunk.
  end = (any @error)?;

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' (('-' @neg)? (digit+ $collect_num)) %/wrap_num <: end;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = ('~' %/wrap_null) end;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.  Also add one for the colon.
  char *pe = p + ( (size < 11) ? size : 11 );
  char *eof = pe;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE mTnetstrings = rb_define_module("TNETS");

  rb_define_method(mTnetstrings, "c_parse", rb_parse_tnets, 1);
  eTnetsParserError = rb_define_class_under(mTnetstrings, "ParserError", rb_eIOError);
}


ifchd: Make the command parser much more strict.  It now has a concept of argument types and command types.

/* ifchd-parse.rl - interface change daemon parser
 *
 * Copyright (c) 2004-2013 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stddef.h>
#include <string.h>

#include "ifchd-defines.h"
#include "log.h"
#include "ifch_proto.h"
#include "strl.h"
#include "linux.h"

%%{
    machine ifchd_parser;

    action Reset { cl->state = STATE_NOTHING; }
    action ArgSt { arg_start = p; }
    action ArgEn {
        arg_len = p - arg_start;
        if (arg_len > sizeof tb - 1) {
            log_line("command argument would overflow");
            return -1;
        }
        memcpy(tb, arg_start, arg_len);
        tb[arg_len] = 0;
    }

    action Dispatch {
        switch (cl->state) {
        case STATE_INTERFACE: perform_interface(cl, tb, arg_len); break;
        case STATE_IP: perform_ip(cl, tb, arg_len); break;
        case STATE_SUBNET: perform_subnet(cl, tb, arg_len); break;
        case STATE_TIMEZONE: perform_timezone(cl, tb, arg_len); break;
        case STATE_ROUTER: perform_router(cl, tb, arg_len); break;
        case STATE_DNS: perform_dns(cl, tb, arg_len); break;
        case STATE_LPRSVR: perform_lprsvr(cl, tb, arg_len); break;
        case STATE_HOSTNAME: perform_hostname(cl, tb, arg_len); break;
        case STATE_DOMAIN: perform_domain(cl, tb, arg_len); break;
        case STATE_IPTTL: perform_ipttl(cl, tb, arg_len); break;
        case STATE_MTU: perform_mtu(cl, tb, arg_len); break;
        case STATE_BROADCAST: perform_broadcast(cl, tb, arg_len); break;
        case STATE_NTPSVR: perform_ntpsrv(cl, tb, arg_len); break;
        case STATE_WINS: perform_wins(cl, tb, arg_len); break;
        default:
            log_line("error: invalid state in dispatch_work");
            return -1;
        }
    }

    terminator = ';' > Dispatch;
    v4addr = digit{1,3} '.' digit{1,3} '.' digit{1,3} '.' digit{1,3};
    ip_arg = (v4addr > ArgSt % ArgEn) terminator;
    iplist_arg = (((v4addr ',')* v4addr) > ArgSt % ArgEn) terminator;
    str_arg = ([^;\0]+ > ArgSt % ArgEn) terminator;
    s32_arg = (extend{4} > ArgSt % ArgEn) terminator;
    u16_arg = (extend{2} > ArgSt % ArgEn) terminator;
    u8_arg = (extend{1} > ArgSt % ArgEn) terminator;

    cmd_ip = ('ip:' % { cl->state = STATE_IP; }
             |'snet:' % { cl->state = STATE_SUBNET; }
             |'routr:' % { cl->state = STATE_ROUTER; }
             |'bcast:' % { cl->state = STATE_BROADCAST; }
             ) ip_arg;
    cmd_iplist = ('dns:' % { cl->state = STATE_DNS; }
                 |'lpr:' % { cl->state = STATE_LPRSVR; }
                 |'ntp:' % { cl->state = STATE_NTPSVR; }
                 |'wins:' % { cl->state = STATE_WINS; }
                 ) iplist_arg;
    cmd_str = ('iface:' % { cl->state = STATE_INTERFACE; }
              |'host:' % { cl->state = STATE_HOSTNAME; }
              |'dom:' % { cl->state = STATE_DOMAIN; }
              ) str_arg;
    cmd_s32 = ('tzone:' % { cl->state = STATE_TIMEZONE; }) s32_arg;
    cmd_u16 = ('mtu:' % { cl->state = STATE_MTU; }) u16_arg;
    cmd_u8  = ('ipttl:' % { cl->state = STATE_IPTTL; }) u8_arg;

    command = (cmd_ip|cmd_iplist|cmd_str|cmd_s32|cmd_u16|cmd_u8);
    main := (command > Reset)+;
}%%

%% write data;

/*
 * Returns -1 on fatal error; that leads to peer connection being closed.
 */
int execute_buffer(struct ifchd_client *cl, char *newbuf)
{
    char buf[MAX_BUF * 2];
    char tb[MAX_BUF];

    if (strnkcpy(buf, cl->ibuf, sizeof buf))
        goto buftooshort;
    if (strnkcat(buf, newbuf, sizeof buf)) {
buftooshort:
        log_line("error: input is too long for buffer");
        return -1;
    }

    size_t init_siz = strlen(buf);
    const char *p = buf;
    const char *pe = p + init_siz;
    const char *arg_start;
    size_t arg_len;
    unsigned int cs = 0;

    %% write init;
    %% write exec;

    size_t bytes_left = pe - p;
    if (bytes_left > 0) {
        size_t taken = init_siz - bytes_left;
        strnkcpy(cl->ibuf, buf + taken, MAX_BUF);
    }

    if (cs < ifchd_parser_first_final) {
        log_line("error: received invalid commands");
        return -1;
    }
    log_line("Commands received and successfully executed.");
    return 0;
}

/* ifchd-parse.rl - interface change daemon parser
 *
 * Copyright (c) 2004-2013 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stddef.h>
#include <string.h>

#include "ifchd-defines.h"
#include "log.h"
#include "ifch_proto.h"
#include "strl.h"
#include "linux.h"

%%{
    machine ifchd_parser;

    action Reset { cl->state = STATE_NOTHING; }
    action ArgSt { arg_start = p; }
    action ArgEn {
        arg_len = p - arg_start;
        if (arg_len > sizeof tb - 1) {
            log_line("command argument would overflow");
            return -1;
        }
        memcpy(tb, arg_start, arg_len);
        tb[arg_len] = 0;
    }

    action Dispatch {
        switch (cl->state) {
        case STATE_INTERFACE: perform_interface(cl, tb, arg_len); break;
        case STATE_IP: perform_ip(cl, tb, arg_len); break;
        case STATE_SUBNET: perform_subnet(cl, tb, arg_len); break;
        case STATE_TIMEZONE: perform_timezone(cl, tb, arg_len); break;
        case STATE_ROUTER: perform_router(cl, tb, arg_len); break;
        case STATE_DNS: perform_dns(cl, tb, arg_len); break;
        case STATE_LPRSVR: perform_lprsvr(cl, tb, arg_len); break;
        case STATE_HOSTNAME: perform_hostname(cl, tb, arg_len); break;
        case STATE_DOMAIN: perform_domain(cl, tb, arg_len); break;
        case STATE_IPTTL: perform_ipttl(cl, tb, arg_len); break;
        case STATE_MTU: perform_mtu(cl, tb, arg_len); break;
        case STATE_BROADCAST: perform_broadcast(cl, tb, arg_len); break;
        case STATE_NTPSVR: perform_ntpsrv(cl, tb, arg_len); break;
        case STATE_WINS: perform_wins(cl, tb, arg_len); break;
        default:
            log_line("error: invalid state in dispatch_work");
            return -1;
        }
    }

    interface = 'iface';
    ip = 'ip';
    subnet = 'snet';
    dns = 'dns';
    lprsvr = 'lpr';
    ntpsvr = 'ntp';
    wins = 'wins';
    router = 'routr';
    broadcast = 'bcast';
    timezone = 'tzone';
    hostname = 'host';
    domain = 'dom';
    ipttl = 'ipttl';
    mtu = 'mtu';

    cmdname = (interface % { cl->state = STATE_INTERFACE; }
              |ip % { cl->state = STATE_IP; }
              |subnet % { cl->state = STATE_SUBNET; }
              |dns % { cl->state = STATE_DNS; }
              |lprsvr % { cl->state = STATE_LPRSVR; }
              |ntpsvr % { cl->state = STATE_NTPSVR; }
              |wins % { cl->state = STATE_WINS; }
              |router % { cl->state = STATE_ROUTER; }
              |broadcast % { cl->state = STATE_BROADCAST; }
              |timezone % { cl->state = STATE_TIMEZONE; }
              |hostname % { cl->state = STATE_HOSTNAME; }
              |domain % { cl->state = STATE_DOMAIN; }
              |ipttl % { cl->state = STATE_IPTTL; }
              |mtu % { cl->state = STATE_MTU; }
              );

    command = cmdname ':' ([^;\0]+ > ArgSt % ArgEn) (';' > Dispatch);
    main := (command > Reset)+;
}%%

%% write data;

/*
 * Returns -1 on fatal error; that leads to peer connection being closed.
 */
int execute_buffer(struct ifchd_client *cl, char *newbuf)
{
    char buf[MAX_BUF * 2];
    char tb[MAX_BUF];

    if (strnkcpy(buf, cl->ibuf, sizeof buf))
        goto buftooshort;
    if (strnkcat(buf, newbuf, sizeof buf)) {
buftooshort:
        log_line("error: input is too long for buffer");
        return -1;
    }

    size_t init_siz = strlen(buf);
    const char *p = buf;
    const char *pe = p + init_siz;
    const char *arg_start;
    size_t arg_len;
    unsigned int cs = 0;

    %% write init;
    %% write exec;

    size_t bytes_left = pe - p;
    if (bytes_left > 0) {
        size_t taken = init_siz - bytes_left;
        strnkcpy(cl->ibuf, buf + taken, MAX_BUF);
    }

    if (cs < ifchd_parser_first_final) {
        log_line("error: received invalid commands");
        return -1;
    }
    log_line("Commands received and successfully executed.");
    return 0;
}


Fix memory leak in JSONPath lexer

Fix memory leak in JSONPath lexer

/**
 * Copyright (c) 2016 Xavier R. Guerin
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
 * to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <ace/tree/Lexer.h>
#include <iostream>
#include <stdexcept>
#include <string>
#include <sstream>
#include <stdlib.h>

#define NEVER(__arg__) __arg__
#include "Parser.c"

namespace {

Token *
getIndex(const char * const beg, const char * const end) {
  size_t result = 0;
  std::istringstream(std::string(beg).substr(0, end-beg)) >> result;
  return new IndexToken { result };
}

Token *
getMember(const char * const beg, const char * const end) {
  return new MemberToken { std::string(beg).substr(0, end-beg) };
}

} // namespace

%%{

machine JSONPath;

action groot_tok {
  Parse(lparser, GROOT, 0, nullptr);
}

action lroot_tok {
  Parse(lparser, LROOT, 0, nullptr);
}

action dot_tok {
  Parse(lparser, DOT, 0, nullptr);
}

action openc_tok {
  Parse(lparser, OPENC, 0, nullptr);
}

action closec_tok {
  Parse(lparser, CLOSEC, 0, nullptr);
}

action slice_tok {
  Parse(lparser, SLICE, 0, nullptr);
}

action wildcard_tok {
  Parse(lparser, WILDCARD, 0, nullptr);
}

action index_tok {
  Parse(lparser, INDEX, getIndex(ts, te), nullptr);
}

action comma_tok {
  Parse(lparser, COMMA, 0, nullptr);
}

action member_tok {
  Parse(lparser, MEMBER, getMember(ts, te), nullptr);
}

groot     = '$';
lroot     = '@';
dot       = '.';
openc     = '[';
closec    = ']';
slice     = ':';
wildcard  = '*';
index     = [0-9]+;
comma     = ',';
member    = [a-zA-Z_\-][a-zA-Z0-9_\-]*;

main := |*
  groot     => groot_tok;
  lroot     => lroot_tok;
  dot       => dot_tok;
  openc     => openc_tok;
  closec    => closec_tok;
  slice     => slice_tok;
  wildcard  => wildcard_tok;
  index     => index_tok;
  comma     => comma_tok;
  member    => member_tok;
*|;

}%%

%% write data;

namespace ace {
namespace tree {
namespace path {

Scan::Scan()
  : cs(0)
  , act(0)
  , ts(0)
  , te(0)
  , lparser(0) {
  lparser = ParseAlloc(malloc);
  %% write init;
}

Scan::~Scan() {
  ParseFree(lparser, free);
}

Path
Scan::parse(std::string const & str) {
  const char* p = str.c_str();
  const char* pe = str.c_str() + str.length();
  const char* eof = pe;
  %% write exec;
  ace::tree::Path * path = nullptr;
  Parse(lparser, 0, 0, &path);
  if (path == nullptr) throw std::invalid_argument(str);
  ace::tree::Path result(*path);
  delete path;
  return result;
}

} // namespace path
} // namespace tree
} // namespace ace

/**
 * Copyright (c) 2016 Xavier R. Guerin
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
 * to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <ace/tree/Lexer.h>
#include <iostream>
#include <stdexcept>
#include <string>
#include <sstream>
#include <stdlib.h>

#define NEVER(__arg__) __arg__
#include "Parser.c"

namespace {

Token *
getIndex(const char * const beg, const char * const end) {
  size_t result = 0;
  std::istringstream(std::string(beg).substr(0, end-beg)) >> result;
  return new IndexToken { result };
}

Token *
getMember(const char * const beg, const char * const end) {
  return new MemberToken { std::string(beg).substr(0, end-beg) };
}

} // namespace

%%{

machine JSONPath;

action groot_tok {
  Parse(lparser, GROOT, 0, nullptr);
}

action lroot_tok {
  Parse(lparser, LROOT, 0, nullptr);
}

action dot_tok {
  Parse(lparser, DOT, 0, nullptr);
}

action openc_tok {
  Parse(lparser, OPENC, 0, nullptr);
}

action closec_tok {
  Parse(lparser, CLOSEC, 0, nullptr);
}

action slice_tok {
  Parse(lparser, SLICE, 0, nullptr);
}

action wildcard_tok {
  Parse(lparser, WILDCARD, 0, nullptr);
}

action index_tok {
  Parse(lparser, INDEX, getIndex(ts, te), nullptr);
}

action comma_tok {
  Parse(lparser, COMMA, 0, nullptr);
}

action member_tok {
  Parse(lparser, MEMBER, getMember(ts, te), nullptr);
}

groot     = '$';
lroot     = '@';
dot       = '.';
openc     = '[';
closec    = ']';
slice     = ':';
wildcard  = '*';
index     = [0-9]+;
comma     = ',';
member    = [a-zA-Z_\-][a-zA-Z0-9_\-]*;

main := |*
  groot     => groot_tok;
  lroot     => lroot_tok;
  dot       => dot_tok;
  openc     => openc_tok;
  closec    => closec_tok;
  slice     => slice_tok;
  wildcard  => wildcard_tok;
  index     => index_tok;
  comma     => comma_tok;
  member    => member_tok;
*|;

}%%

%% write data;

namespace ace {
namespace tree {
namespace path {

Scan::Scan()
  : cs(0)
  , act(0)
  , ts(0)
  , te(0)
  , lparser(0) {
  lparser = ParseAlloc(malloc);
  %% write init;
}

Scan::~Scan() {
  ParseFree(lparser, free);
}

Path
Scan::parse(std::string const & str) {
  const char* p = str.c_str();
  const char* pe = str.c_str() + str.length();
  const char* eof = pe;
  %% write exec;
  ace::tree::Path * path = nullptr;
  Parse(lparser, 0, 0, &path);
  if (path == nullptr) throw std::invalid_argument(str);
  ace::tree::Path::Ref pref(path);
  return *path;
}

} // namespace path
} // namespace tree
} // namespace ace