Don't need uri.rl anymore

Don't need uri.rl anymore

%%{
  machine uri;

  # === URI spec
  delims        = ( "<" | ">" | "#" | "%" | "\"" );
  unwise        = ( "{" | "}" | "|" | "^" | "[" | "]" | "`" | "\\" );
  mark          = ( "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" );
  reserved      = ( ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," );
  escaped       = ( "%" "u"? xdigit xdigit ); # The optional u is to handle an IE 7 bug
  unreserved    = ( alnum | mark );
  ie6_hax       = ( "\"" | "<" | ">" );
  uric          = ( reserved | unreserved | escaped | ie6_hax );
  pchar         = ( unreserved | escaped | ie6_hax | ":" | "@" | "&" | "=" | "+" | "$" | "," );
  opaque        = ( uric -- "/" ) uric *;

  fragment      = ( uric * );
  query         = ( uric * ) > start_query % end_query;
  param         = ( pchar * );
  segment       = ( pchar * ( ";" param ) * );
  segments      = ( segment ( "/" segment ) * );
  abs_path      = ( "/" segments ) > start_path % end_path;
  path          = ( abs_path | opaque ) ?;

  port          = ( digit ) *;
  ipv4addr      = ( digit + "." digit + "." digit + "." digit + );
  toplabel      = ( alpha | alpha ( alnum | "-" ) * alnum );
  domainlabel   = ( alnum | alnum ( alnum | "-" ) * alnum );
  hostname      = ( domainlabel "." ) * toplabel "." ?;
  host          = ( hostname | ipv4addr );
  hostport      = ( host ( ":" port ) ? );

  userinfo      = ( unreserved | escaped | ";" | ":" | "&" | "=" | "+" | "$" | "," ) *;
  server        = ( ( userinfo "@" ) ? hostport ) ?;
  reg_name      = ( unreserved | escaped | "$" | "," | ";" | ":" | "@" | "&" | "=" | "+" ) +;
  authority     = ( server | reg_name );
  scheme        = ( alpha ( alnum | "+" | "-" | "." ) * );
  rel_segment   = ( unreserved | escaped | ";" | "@" | "&" | "=" | "+" | "$" | "," ) + ;
  rel_path      = ( rel_segment abs_path ? );
  net_path      = ( "//" authority abs_path ? );
  hier_part     = ( net_path | abs_path );

  relative_uri  = ( net_path | abs_path | rel_path ) ( "?" query ) ?;
  absolute_uri  = ( scheme ":" ( ( hier_part ( "?" query ) ? ) | opaque ) );
  uri           = ( absolute_uri | relative_uri ) ( "#" fragment ) ?;
}%%


condensed nullptr checks

condensed nullptr checks

#include "hext/lexer.h"

namespace hext {

lexer::lex_error::lex_error(const std::string& msg)
: std::runtime_error(msg)
{
}

lexer::lexer(const char * begin, const char * end)
: p_begin(begin),
  p(begin),
  pe(end),
  eof(end),
  cs(0)
{
  using namespace ragel;
  %%{
    machine hext;
    include "hext.rl";
    write init;
  }%%
}

std::vector<token> lexer::lex()
{
  using namespace ragel;

  std::vector<token> tokens;

  // this calls throw_error on lexing error
  %% write exec;

  return tokens;
}

void lexer::throw_error() const
{
  std::pair<size_t, size_t> pos =
    get_char_position(this->p, this->p_begin, this->pe);

  std::string char_name;
  if( this->p == this->pe )
    char_name = "[eof]";
  else if( this->p )
    char_name = get_char_name(*this->p);
  else
    char_name = "[unknown]";

  std::stringstream error_msg;
  error_msg << "Error at line "
            << pos.first + 1
            << ", char "
            << pos.second + 1
            << ", unexpected character '"
            << char_name
            << "'";

  throw lex_error(error_msg.str());
}

} // namespace hext

#include "hext/lexer.h"

namespace hext {

lexer::lex_error::lex_error(const std::string& msg)
: std::runtime_error(msg)
{
}

lexer::lexer(const char * begin, const char * end)
: p_begin(begin),
  p(begin),
  pe(end),
  eof(end),
  cs(0)
{
  using namespace ragel;
  %%{
    machine hext;
    include "hext.rl";
    write init;
  }%%
}

std::vector<token> lexer::lex()
{
  using namespace ragel;

  std::vector<token> tokens;

  // this calls throw_error on lexing error
  %% write exec;

  return tokens;
}

void lexer::throw_error() const
{
  std::pair<size_t, size_t> pos =
    get_char_position(this->p, this->p_begin, this->pe);

  std::string char_name;
  if( this->p == this->pe )
  {
    char_name = "[eof]";
  }
  else
  {
    if( this->p != nullptr )
      char_name = get_char_name(*this->p);
    else
      char_name = "[unknown]";
  }

  std::stringstream error_msg;
  error_msg << "Error at line "
            << pos.first + 1
            << ", char "
            << pos.second + 1
            << ", unexpected character '"
            << char_name
            << "'";

  throw lex_error(error_msg.str());
}

} // namespace hext


Mostly working mb scanner, still failing on long words

git-svn-id: 26a6d2a58b162bc133329b2f36847a121ee54040@978 06fd6eb0-0002-0410-a719-e5602cce40bc

/* scanner.rl -*-C-*- */
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>
#include "global.h"
#include "internal.h"

static char *position_in_mb( const unsigned long *orig_wc,
                             const char *orig_mb,
                             const unsigned long *curr_wc )
{
    char          *mb = (char *)orig_mb;
    unsigned long *wc = (unsigned long *)orig_wc;

    while (wc < curr_wc)
    {
        char buf[MB_LEN_MAX];
        mb += wctomb(buf, *wc);
        ++wc;
    }

    return mb;
}

#define RET goto ret;

#define STRIP(c) do { \
    strip_char = c;   \
    goto strip;       \
} while(0)

%%{
    machine StdTokMb;
    alphtype unsigned long;
    include WChar "src/wchar.rl";

    delim = space;
    token = walpha walnum*;
    punc  = [.,\/_\-];
    proto = 'http'[s]? | 'ftp' | 'file';
    urlc  = walnum | punc | [\@\:];

    main := |*

        #// Token, or token with possessive
        token           { RET; };
        token [\']      { trunc = 1; RET; };
        token [\'][sS]? { trunc = 2; RET; };

        #// Token with hyphens
        walnum+ ('-' walnum+)* { RET; };

        #// Company name
        token [\&\@] token* { RET; };

        #// URL
        proto [:][/]+ %{ skip = p - ts; } urlc+ [/] { trunc = 1; RET; };
        proto [:][/]+ %{ skip = p - ts; } urlc+     { RET; };
        walnum+[:][/]+ urlc+ [/] { trunc = 1; RET; };
        walnum+[:][/]+ urlc+     { RET; };

        #// Email
        walnum+ '@' walnum+ '.' walpha+ { RET; };

        #// Acronym
        (walpha '.')+ walpha { STRIP('.'); };

        #// Int+float
        [\-\+]?wdigit+            { RET; };
        [\-\+]?wdigit+ '.' wdigit+ { RET; };

        #// Ignore whitespace and other crap
        0 { return; };
        (any - walnum);

        *|;
}%%

%% write data nofinal;

static int mb_next_char(unsigned long *wchr, const char *s, mbstate_t *state)
{
    int num_bytes;
    if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
        const char *t = s;
        do {
            t++;
            ZEROSET(state, mbstate_t);
            num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
        } while ((num_bytes < 0) && (*t != 0));
        num_bytes = t - s;
        if (*t == 0) *wchr = 0;
    }
    return num_bytes;
}

static int wc_next_char(char *s, const unsigned long *wchr, mbstate_t *state)
{
    return (int)wcrtomb(s, *wchr, state);
}

/*
 * All input to Ragel must be in a buffer of 32bit unicode codepoints.
 * To that end, we require that the input to the scanner must be in a
 * codepage that mbtowc will convert to unicode codepoints.  The easy
 * way to do this is to supply the scanner with UTF8.  It will call
 * mbtowc on the buffer, pass it to the tokenizer which will extract
 * one token out.  This token will then be converted back with wctomb.
 *
 * frt_std_scan_mb takes in a pointer to the mb buffer +inmb+ that has
 * max size +in_size+.  The resulting token will be stored in +outmb+,
 * with at most +out_size+ bytes written.
 *
 * While tokenizing, part of the token may be stripped out. Eg,
 * 'foo!!' will become 'foo', and 'http://www.bar.com' will become
 * 'www.bar.com'.  So that the caller can track what exactly has been
 * tokenized in +inmb+, +startmb+ and +endmb+ are set to point to the
 * extremeties of the original unmodified token in +inmb+.
 *
 * The size of the token written out to +outmb+ is stored in
 * +token_size+.
 */

static void mb_to_wc(const char *in, size_t in_size,
                     unsigned long *out, size_t out_size)
{
    mbstate_t state;
    char *in_p           = (char *)in;
    unsigned long *out_p = out;
    ZEROSET(&state, mbstate_t);

    while (in_p < (in + in_size) && out_p < (out + out_size/sizeof(*out)))
    {
        if (!*in_p)
            break;

        int n = mb_next_char(out_p, in_p, &state);
        if (n < 0)
        {
            ++in_p;
            continue;
        }

        /* We can break out early here on, say, a space XXX */
        in_p += n;
        ++out_p;
    }
}

static void wc_to_mb(char *out, size_t out_size, int *token_size,
                     const unsigned long *in_wc, size_t in_wc_size)
{
    mbstate_t state;
    char *out_p = out;
    const unsigned long *in_wc_p = in_wc;
    ZEROSET(&state, mbstate_t);
    *token_size = 0;

    while (out_p < (out + out_size) && in_wc_p < (in_wc + in_wc_size))
    {
        if (!*in_wc_p)
            break;

        int n = wc_next_char(out_p, in_wc_p, &state);
        if (n < 0)
        {
            ++in_wc_p;
            continue;
        }

        out_p += n;
        ++in_wc_p;
    }

    *token_size = out_p - out;
}

void frt_std_scan_mb(const char *in_mb, size_t in_mb_size,
                 char *out_mb, size_t out_mb_size,
                 char **start_mb, char **end_mb,
                 int *token_size)
{
    int cs, act;
    unsigned long *ts = 0, *te = 0;

    %% write init;

    unsigned long in_wc[4096] = {0};
    unsigned long out_wc[4096] = {0};
    mb_to_wc(in_mb, in_mb_size, in_wc, sizeof(in_wc));

    unsigned long *p = in_wc;
    unsigned long *pe = 0;
    unsigned long *eof = pe;
    int skip = 0;
    int trunc = 0;
    unsigned long strip_char = 0;

    *end_mb = 0;
    *start_mb = 0;
    *token_size = 0;

    %% write exec;

    if ( cs == StdTokMb_error )
                   fwprintf(stderr, L"PARSE ERROR\n");
    else if ( ts ) fwprintf(stderr, L"STUFF LEFT: '%ls'\n", ts);
    return;

 ret:
    {
        *start_mb   = position_in_mb(in_wc, in_mb, ts);
        *end_mb     = position_in_mb(in_wc, in_mb, te);

        size_t __len = te - ts - skip - trunc;
        memcpy(out_wc, ts + skip, __len*sizeof(unsigned long));

        wc_to_mb(out_mb, out_mb_size, token_size, out_wc, sizeof(out_wc));
        out_mb[*token_size] = 0;
        return;
    }

 strip:
    {
        *start_mb = position_in_mb(in_wc, in_mb, ts);
        *end_mb   = position_in_mb(in_wc, in_mb, te);

        size_t __len = te - ts - skip - trunc;
        unsigned long *__p = ts + skip;
        unsigned long *__o = out_wc;
        for (; __p < (ts + skip + __len); ++__p) {
            if (*__p != strip_char)
                *__o++ = *__p;
        }

        wc_to_mb(out_mb, out_mb_size, token_size, out_wc, sizeof(out_wc));
        out_mb[*token_size] = 0;
    }
}

/* scanner.rl -*-C-*- */
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>
#include "global.h"
#include "internal.h"

static char *position_in_mb( const unsigned long *orig_wc,
                             const char *orig_mb,
                             const unsigned long *curr_wc )
{
    char          *mb = (char *)orig_mb;
    unsigned long *wc = (char *)orig_wc;

    while (wc < curr_wc)
    {
        char buf[MB_LEN_MAX];
        mb += wctomb(buf, *wc);
        ++wc;
    }

    return mb;
}

#define RET do {                        \
    *start   = position_in_mb(buf, in, ts); \
    *end     = position_in_mb(buf, in, te); \
    size_t __len = *end - *start - skip - trunc; \
    *len     = __len;                   \
    if (__len > out_size)               \
        __len = out_size;               \
     memcpy(out, *start + skip, __len);     \
     out[__len] = 0;                    \
     return;                            \
} while(0)

#define STRIP(c) do {                         \
    unsigned long *__p = ts;                  \
    unsigned long *__o = out;                 \
    unsigned long *__max = __p + out_size;    \
    for (; __p <= te && __p < __max; ++__p) { \
        if (*__p != c)                        \
            *__o++ = *__p;                    \
    }                                         \
    *__o = 0;                                 \
                                              \
    *start = position_in_mb(buf, in, ts);                              \
    *end   = position_in_mb(buf, in, te);                              \
    *len   = (char *)__o - (char *)out;                       \
    return;                                   \
} while(0)

%%{
    machine StdTokMb;
    alphtype unsigned long;
    include WChar "src/wchar.rl";

    delim = space;
    token = walpha walnum*;
    punc  = [.,\/_\-];
    proto = 'http'[s]? | 'ftp' | 'file';
    urlc  = walnum | punc | [\@\:];

    main := |*

        #// Token, or token with possessive
        token           { RET; };
        token [\']      { trunc = 1; RET; };
        token [\'][sS]? { trunc = 2; RET; };

        #// Token with hyphens
        walnum+ ('-' walnum+)* { RET; };

        #// Company name
        token [\&\@] token* { RET; };

        #// URL
        proto [:][/]+ %{ skip = p - ts; } urlc+ [/] { trunc = 1; RET; };
        proto [:][/]+ %{ skip = p - ts; } urlc+     { RET; };
        walnum+[:][/]+ urlc+ [/] { trunc = 1; RET; };
        walnum+[:][/]+ urlc+     { RET; };

        #// Email
        walnum+ '@' walnum+ '.' walpha+ { RET; };

        #// Acronym
        (walpha '.')+ walpha { STRIP('.'); };

        #// Int+float
        [\-\+]?wdigit+            { RET; };
        [\-\+]?wdigit+ '.' wdigit+ { RET; };

        #// Ignore whitespace and other crap
        0 { return; };
        (any - walnum);

        *|;
}%%

%% write data nofinal;

static int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
{
    int num_bytes;
    if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
        const char *t = s;
        do {
            t++;
            ZEROSET(state, mbstate_t);
            num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
        } while ((num_bytes < 0) && (*t != 0));
        num_bytes = t - s;
        if (*t == 0) *wchr = 0;
    }
    return num_bytes;
}

/* Function takes in a multibyte string, converts it to wide-chars and
   then tokenizes that. */
void frt_std_scan_mb(const char *in, size_t in_size,
                 char *out, size_t out_size,
                 char **start, char **end,
                 int *len)
{
    int cs, act;
    unsigned long *ts, *te = 0;

    %% write init;

    unsigned long buf[4096] = {0};
    unsigned long *bufp = buf;
    char *inp = (char *)in;
    mbstate_t state;
    ZEROSET(&state, mbstate_t);
    printf ("TRYING TO PARSE: '%s'\n", in);
    while (inp < (in + in_size) && bufp < (buf + sizeof(buf)/sizeof(*buf)))
    {
        if (!*inp)
            break;

        int n = mb_next_char((wchar_t *)bufp, inp, &state);
        if (n < 0)
        {
            printf ("Error parsing input\n");
            ++inp;
            continue;
        }

        /* We can break out early here on, say, a space XXX */
        inp += n;
        ++bufp;
    }

    printf ("parsed: %d\n", inp - in);
    wprintf (L"%ls\n", buf);
    printf ("%04x\n", buf[5]);

    unsigned long *p = (unsigned long *)buf;
    unsigned long *pe = 0;
    unsigned long *eof = pe;
    *len = 0;
    int skip = 0;
    int trunc = 0;

    %% write exec;

    if ( cs == StdTokMb_error )
    {
        fprintf(stderr, "PARSE ERROR\n" );
        return;
    }

    if ( ts )
    {
        fwprintf(stderr, L"STUFF LEFT: '%ls'\n", ts);
    }
}


Ragel simplify

Ragel simplify

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )+;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \"HGVS\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s) }

cmd=ARGV.shift
run_lexer(cmd)

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )+;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \"HGVS\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s) }

cmd=ARGV.shift
run_lexer(cmd)


[confscanner] removed out-dated TODO

[confscanner] removed out-dated TODO

#include "config.h"
#include "confscanner.h"

#ifdef WIN32
	#include <windows.h>
	#include <shlwapi.h>
	#pragma comment (lib, "shlwapi.lib")
#else
	extern "C" {
		#include <fnmatch.h>
	}
#endif

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      //cerr << "glob matched!" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "adding: " << tmp_string.substr(0, found-1) + '\n';
          //cerr << endl;
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          /*if(!param_list_["grid"].empty())
            cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];*/
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
/*    cerr << "register glob: " << tmp_string << endl;
    cerr << " on top of globlist_stack " << endl;*/
    //cerr_stack();
//    cerr << "\n";
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{cblock_ = 0; fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                {
                  if(cblock_ != 0)
                    (*cblock_).clear();
                }
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list:\n"; cerr.write(p, 10); cerr << endl;*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2\n"; cerr.write(p, 20); cerr << endl;*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (MIDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok )
              . (MWSOC*
                 . (GLOB)
                )*
              . MWSOC*
              . ';'
                @{
                  arglist_ = &vars_[tmp_string];
                  arglist_->push_back(std::string(tmp_p, p - tmp_p));
                 }
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string is matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    const char * glob = (*it).c_str();
#ifndef WIN32
    int flags = 0;
#endif
    if (glob[0] == '.' && glob[1] == '/')
    {
      glob = &glob[2];
#ifndef WIN32
      flags = FNM_PATHNAME;
#endif
    }

    // check wether str matches glob
#ifdef WIN32
    if(PathMatchSpec(str,glob))
#else
    if(fnmatch(glob, str, flags) == 0)
#endif
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename is matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
  {
    //cerr << "matched " << s << endl;
    return true;
  }

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
    {
      //cerr << "matched " << s << endl;
      return true;
    }

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
    {
      //cerr << "matched " << s << endl;
      return true;
    }
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

/**
 * @class ConfFileScanner
 *
 * @change{1,2,dw,2011-11-07} Giving correct error message when no configuration file is given or found.
 */
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : buf(new char[BUFSIZE]), line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "./mtocpp.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  //cerr << "filename" << filename << endl;
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "mtoc++ No configuration file specified or file not found. Using default values." << endl;
    conffile_ = "";
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  if ( ! conffile_.empty() )
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    /* Do the first read. */
    bool done = false;
    while ( !done )
    {
      char *p = buf + have;
      char *tmp_p = p;// *tmp_p2 = p;
      // string for temporary tokens
      string tmp_string;
      // spare space in buffer
      int space = BUFSIZE - have;

      if ( space == 0 )
      {
        /* We filled up the buffer trying to scan a token. */
        cerr << "OUT OF BUFFER SPACE" << endl;
        exit(1);
      }

      // read configuration file chunk in buffer
      confistream_.read( p, space );
      int len = confistream_.gcount();
      char *pe = p + len;
      char *rpe = pe;

      /* If we see eof then append the EOF char. */
      if ( confistream_.eof() )
      {
        done = true;
      }
      else
      {
        /* Find the last semicolon by searching backwards. This
         * is where we will stop processing on this iteration. */
        while ( *pe!= ';' && pe > p )
          pe--;
      }

      //std::cerr << "execute parser" << std::endl;
      %% write exec;
      //std::cerr << "finish parser" << std::endl;

      /* Check if we failed. */
      if ( cs == ConfFileScanner_error )
      {
        /* Machine failed before finding a token. */
        cerr << "PARSE ERROR in mtoc++ config file, line " << line << endl;
        exit(1);
      }

      have = rpe - pe;
      /* cerr << "memmove by " << have << "bytes\n";*/
      memmove( buf, pe, have );
    }
    //cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "config.h"
#include "confscanner.h"

#ifdef WIN32
	#include <windows.h>
	#include <shlwapi.h>
	#pragma comment (lib, "shlwapi.lib")
#else
	extern "C" {
		#include <fnmatch.h>
	}
#endif

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      //cerr << "glob matched!" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "adding: " << tmp_string.substr(0, found-1) + '\n';
          //cerr << endl;
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          /*if(!param_list_["grid"].empty())
            cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];*/
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
/*    cerr << "register glob: " << tmp_string << endl;
    cerr << " on top of globlist_stack " << endl;*/
    //cerr_stack();
//    cerr << "\n";
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{cblock_ = 0; fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                {
                  if(cblock_ != 0)
                    (*cblock_).clear();
                }
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list:\n"; cerr.write(p, 10); cerr << endl;*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2\n"; cerr.write(p, 20); cerr << endl;*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (MIDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok )
              . (MWSOC*
                 . (GLOB)
                )*
              . MWSOC*
              . ';'
                @{
                  arglist_ = &vars_[tmp_string];
                  arglist_->push_back(std::string(tmp_p, p - tmp_p));
                 }
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string is matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    const char * glob = (*it).c_str();
#ifndef WIN32
    int flags = 0;
#endif
    if (glob[0] == '.' && glob[1] == '/')
    {
      glob = &glob[2];
#ifndef WIN32
      flags = FNM_PATHNAME;
#endif
    }

    // check wether str matches glob
#ifdef WIN32
    if(PathMatchSpec(str,glob))
#else
    if(fnmatch(glob, str, flags) == 0)
#endif
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename is matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
  {
    //cerr << "matched " << s << endl;
    return true;
  }

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
    {
      //cerr << "matched " << s << endl;
      return true;
    }

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
    {
      //cerr << "matched " << s << endl;
      return true;
    }
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

/**
 * @class ConfFileScanner
 *
 * @change{1,2,dw,2011-11-07} Giving correct error message when no configuration file is given or found.
 */
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : buf(new char[BUFSIZE]), line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "./mtocpp.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  //cerr << "filename" << filename << endl;
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "mtoc++ No configuration file specified or file not found. Using default values." << endl;
    conffile_ = "";
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  if ( ! conffile_.empty() )
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    /* Do the first read. */
    bool done = false;
    while ( !done )
    {
      char *p = buf + have;
      char *tmp_p = p;// *tmp_p2 = p;
      // string for temporary tokens
      string tmp_string;
      // spare space in buffer
      int space = BUFSIZE - have;

      if ( space == 0 )
      {
        /* We filled up the buffer trying to scan a token. */
        cerr << "OUT OF BUFFER SPACE" << endl;
        exit(1);
      }

      // read configuration file chunk in buffer
      confistream_.read( p, space );
      int len = confistream_.gcount();
      char *pe = p + len;
      char *rpe = pe;

      /* If we see eof then append the EOF char. */
      if ( confistream_.eof() )
      {
        done = true;
      }
      else
      {
        /* Find the last semicolon by searching backwards. This
         * is where we will stop processing on this iteration. */
        while ( *pe!= ';' && pe > p )
          pe--;
      }

      //std::cerr << "execute parser" << std::endl;
      %% write exec;
      //std::cerr << "finish parser" << std::endl;

      /* Check if we failed. */
      if ( cs == ConfFileScanner_error )
      {
        /* Machine failed before finding a token. */
        cerr << "PARSE ERROR in mtoc++ config file, line " << line << endl;
        exit(1);
      }

      have = rpe - pe;
      /* cerr << "memmove by " << have << "bytes\n";*/
      memmove( buf, pe, have );
    }
    //cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */


this really allows Daniel's generic typenames

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('<' . [^&]* . '>') => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('>,') => { fout << ">"; --p; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */