fix rules and counting, cleanup

fix rules and counting, cleanup

* IRIs are required to be absolute (have a scheme)
* Handle blank node as object followed by no whitespace and '.'
* Comments are also allowed after a triple
* Count lines skipped due to failure
* Refactor common expressions in rules
* Avoid the need for a stack by using fgoto instead of fcall

#include <string.h>

#include "parser.h"
#include "parser_internal.h"

#define BUFSIZE 0x10000

// TODO:
// - error handling, also when string methods fail

unsigned int parse_hex(const unsigned char *src, unsigned int len) {
    unsigned int i = 0;
    for (unsigned int j = 0; j < len; j++) {
        i *= 16;
        char c = src[j];
        if (c >= '0' && c <= '9') {
            i += c - '0';
        } else if ((c >= 'A' && c <= 'F')
                   || (c >= 'A' && c <= 'F'))
        {
            i += c - 'A' + 10;
        } else if (c >= 'a' && c <= 'f') {
            i += c - 'a' +  10;
        }
    }
    return i;
}

void emit(entee_parser *parser, entee_token_type type) {
    if (!string_end(parser->string)) {
        parser->finished = 1;
    } else {
        parser->handler(type, parser->string->chars, parser->handler_arg);
    }
}

%%{
    machine ntriples;

    alphtype unsigned char;

    action mark { mark = p; }

    action unescape {
        unsigned int codepoint = parse_hex(mark, p - mark);
        mark = 0;
        if (!string_add_codepoint(parser->string, codepoint)) {
            parser->finished = 1;
        }
    }

    action endTriple {
        // TODO: call handler?
    }

    action endLine {
        parser->line++;
    }

    action putChar {
        if (!string_add_checked(parser->string, *p))
            parser->finished = 1;
    }

    action startString {
        if (parser->string != NULL) {
            string_free(parser->string);
        }
        parser->string = string_new();
        if (parser->string == NULL) {
            parser->finished = 1;
        }
    }

    action endSimpleLiteral {
        emit(parser, ENTEE_SIMPLE_LITERAL_VALUE);
    }

    action datatypeLiteral {
        emit(parser, ENTEE_DATATYPE_LITERAL_VALUE);
    }

    action langtagLiteral {
        emit(parser, ENTEE_LANGUAGE_TAGGED_LITERAL_VALUE);
    }

    action endLangtagLiteral {
        emit(parser, ENTEE_LANGUAGE_TAGGED_LITERAL_LANGUAGE);
    }

    action endDatatypeLiteral {
        emit(parser, ENTEE_DATATYPE_LITERAL_IRI);
    }

    action endIRI {
        emit(parser, ENTEE_IRI);
    }

    action endBlankNode {
        emit(parser, ENTEE_BLANK_NODE);
    }

    WS = ' ' | '\t';
    EOL = '\r' | '\n';
    ALPHA = 'A' .. 'Z'
          | 'a' .. 'z';
    DIGIT = '0' .. '9';
    HEX = DIGIT | 'A' .. 'F' | 'a' .. 'f';
    PN_CHARS_BASE = (ALPHA
                     # The N-Triple spec specifies Unicode codepoints,
                     # but files are encoded (and parsed) as UTF-8:
                     # 0x00C0 .. 0x00D6
                     | 0xC3 0x80..0x96
                     # 0x00D8 .. 0x00F6
                     | 0xC3 0x98..0xB6
                     # 0x00F8 .. 0x02FF
                     | 0xC3 0xB8..0xFF
                     | 0xC4..0xCA 0x00..0xFF
                     | 0xCB 0x00..0xBF
                     # 0x0370 .. 0x037D
                     | 0xCD 0xB0..0xBD
                     # 0x037F .. 0x1FFF = 0x037F .. 0x07FF | 0x0800 .. 0x1FFF
                     | 0xCD 0xBF..0xFF
                     | 0xCE..0xDE 0x00..0xFF
                     | 0xDF 0x00..0xBF
                     | 0xE0 0xA0 0x80..0xFF
                     | 0xE0 0xA1..0xFF 0x00..0xFF
                     | 0xE1 0x00 0x00..0xFF
                     | 0xE1 0x01..0xBE 0x00..0xFF
                     | 0xE1 0xBF 0x00..0xBF
                     # 0x200C .. 0x200D
                     | 0xE2 0x80 0x8C..0x8D
                     # 0x2070 .. 0x218F
                     | 0xE2 0x81 0xB0..0xFF
                     | 0xE2 0x82..0x85 0x00..0xFF
                     | 0xE2 0x86 0x00..0x8F
                     # 0x2C00 .. 0x2FEF
                     | 0xE2 0xB0 0x80..0xFF
                     | 0xE2 0xB1..0xBE 0x00..0xFF
                     | 0xE2 0xBF 0x00..0xAF
                     # 0x3001 .. 0xD7FF
                     | 0xE3 0x80 0x81..0xFF
                     | 0xE3 0x81..0xFF 0x00..0xFF
                     | 0xE4..0xEC 0x00..0xFF 0x00..0xFF
                     | 0xED 0x00 0x00..0xFF
                     | 0xED 0x01..0x9E 0x00..0xFF
                     | 0xED 0x9F 0x00..0xBF
                     # 0xF900 .. 0xFDCF
                     | 0xEF 0xA4 0x80..0xFF
                     | 0xEF 0xA5..0xB6 0x00..0xFF
                     | 0xEF 0xB7 0x00..0x8F
                     # 0xFDF0 .. 0xFFFD
                     | 0xEF 0xB7 0xB0..0xFF
                     | 0xEF 0xB8..0xBE 0x00..0xFF
                     | 0xEF 0xBF 0x00..0xBD
                     # 0x10000 .. 0xEFFFF
                     | 0xF0 0x90 0x80 0x80..0xFF
                     | 0xF0 0x90 0x81..0xFF 0x00..0xFF
                     | 0xF0 0x91..0xFF 0x00..0xFF 0x00..0xFF
                     | 0xF1..0xF2 0x00..0xFF 0x00..0xFF 0x00..0xFF
                     | 0xF3 0x00 0x00 0x00..0xFF
                     | 0xF3 0x00 0x01..0xFF 0x00..0xFF
                     | 0xF3 0x01..0xAE 0x00..0xFF 0x00..0xFF
                     | 0xF3 0xAF 0x00 0x00..0xFF
                     | 0xF3 0xAF 0x01..0xBE 0x00..0xFF
                     | 0xF3 0xAF 0xBF 0x00..0xBF
                     ) $putChar;
    PN_CHARS_U = PN_CHARS_BASE
               | ('_' | ':') $putChar;
    PN_CHARS = PN_CHARS_U
               | ('-' | DIGIT | 0x00B7
                  # 0x0300 .. 0x036F
                  | 0xCC 0x80..0xFF
                  | 0xCD 0x00..0xAF
                  # 0x203F .. 0x2040
                  | 0xE2 0x80 0xBF..0xFF
                  | 0xE2 0x81 0x00..0x80
                  ) $putChar;
    ECHAR = '\\' .
            (  ('t' ${ string_add_checked(parser->string, '\t'); })
             | ('b' ${ string_add_checked(parser->string, '\b'); })
             | ('n' ${ string_add_checked(parser->string, '\n'); })
             | ('r' ${ string_add_checked(parser->string, '\r'); })
             | ('f' ${ string_add_checked(parser->string, '\f'); })
             | ('"' ${ string_add_checked(parser->string, '\"'); })
             | ('\'' ${ string_add_checked(parser->string, '\''); })
             | ('\\' ${ string_add_checked(parser->string, '\\'); })
             );
    UCHAR = '\\' .
            (  ('u' HEX{4} >mark %unescape)
             | ('U' HEX{8} >mark %unescape));
    STRING_VALUE = (^(0x22 | 0x5C | 0xA | 0xD) $putChar
                    | ECHAR
                    | UCHAR)*
                   >startString;
    scheme = (ALPHA (ALPHA | DIGIT | '+' | '-' | '.')*) $putChar;
    IRIREF_VALUE = scheme >startString ':' $putChar
                   (^(0x00 .. 0x20
                      | '<' | '>'
                      | '"' | '{' | '}'
                      | '|' | '^' | '`' | '\\') $putChar
                    | UCHAR)*;
    IRIREF = '<' IRIREF_VALUE '>';
    BLANK_NODE_LABEL = '_:' >startString
                       (PN_CHARS_U | DIGIT $putChar)
                       ((PN_CHARS | '.' $putChar)* PN_CHARS)?;
    LANGTAG_VALUE = ALPHA+ $putChar >startString
                    ('-' $putChar ((ALPHA | DIGIT) $putChar)+)*;
    datatypeLiteral = '"' STRING_VALUE '"' '^^' >datatypeLiteral
                      IRIREF %endDatatypeLiteral;
    langtagLiteral = '"' STRING_VALUE '"' '@' >langtagLiteral
                     LANGTAG_VALUE %endLangtagLiteral;
    simpleLiteral = '"' STRING_VALUE '"' %endSimpleLiteral;
    literal = datatypeLiteral
            | langtagLiteral
            | simpleLiteral;
    iri = IRIREF %endIRI;
    blankNode = BLANK_NODE_LABEL %endBlankNode;
    subject = iri
            | blankNode;
    predicate = iri;
    object = literal
           | iri
           | blankNode;
    triple = subject WS* predicate WS* object WS* :>> '.' WS* %endTriple;
    comment = '#' (any -- EOL)*;
    line = WS* triple? comment? (EOL @endLine)+;
    skip_line := (any -- EOL)* (EOL @endLine) @{
        fgoto main;
    };
    main := line* $err{
        // TODO: failed on line parser->line
        fhold; fgoto skip_line;
    };

    write data;
}%%

entee_parser *entee_new_parser() {
    entee_parser *parser = malloc(sizeof(entee_parser));
    parser->string = NULL;
    parser->line = 1;
    parser->finished = 0;
    parser->reader = NULL;
    parser->reader_arg = NULL;
    parser->handler = NULL;
    parser->handler_arg = NULL;
    return parser;
}

void entee_free_parser(entee_parser *parser) {
    free(parser);
}

void entee_parser_set_reader(entee_parser *parser,
                             entee_reader reader,
                             void *reader_arg)
{
    parser->reader = reader;
    parser->reader_arg = reader_arg;
}

void entee_parser_set_handler(entee_parser *parser,
                              entee_handler handler,
                              void *handler_arg)
{
    parser->handler = handler;
    parser->handler_arg = handler_arg;
}

int entee_parser_parse(entee_parser *parser) {
    if (parser->reader == NULL
        || parser->handler == NULL)
    {
        return 0;
    }

    int cs;

    %% write init;

    int res = 1;
    unsigned char buf[BUFSIZE];
    unsigned char *mark = 0;
    int have = 0;

    entee_reader reader = parser->reader;

    while (!parser->finished) {
        unsigned char *p = buf + have;
        unsigned char *pe;
        unsigned char *eof = 0;

        int space = BUFSIZE - have;
        if (space == 0) {
            // out of buffer space
            res = 0;
            break;
        }

        int len = reader(p, space, parser->reader_arg);
        pe = p + len;

        if (len < space) {
            eof = pe;
            parser->finished = 1;
        }

        %% write exec;

        if (cs == ntriples_error) {
            res = 0;
            break;
        } else if (mark) {
            have = pe - mark;
            memmove(buf, mark, have);
            mark = buf;
        } else {
            have = 0;
        }
    }

    return res;
}

#include <string.h>

#include "parser.h"
#include "parser_internal.h"

#define BUFSIZE 0x10000

// TODO:
// - error handling, also when string methods fail

unsigned int parse_hex(const unsigned char *src, unsigned int len) {
    unsigned int i = 0;
    for (unsigned int j = 0; j < len; j++) {
        i *= 16;
        char c = src[j];
        if (c >= '0' && c <= '9') {
            i += c - '0';
        } else if ((c >= 'A' && c <= 'F')
                   || (c >= 'A' && c <= 'F'))
        {
            i += c - 'A' + 10;
        } else if (c >= 'a' && c <= 'f') {
            i += c - 'a' +  10;
        }
    }
    return i;
}

void emit(entee_parser *parser, entee_token_type type) {
    if (!string_end(parser->string)) {
        parser->finished = 1;
    } else {
        parser->handler(type, parser->string->chars, parser->handler_arg);
    }
}

%%{
    machine ntriples;

    alphtype unsigned char;

    action mark { mark = p; }

    action unescape {
        unsigned int codepoint = parse_hex(mark, p - mark);
        mark = 0;
        if (!string_add_codepoint(parser->string, codepoint)) {
            parser->finished = 1;
        }
    }

    action endTriple {
        // TODO: call handler?
    }

    action endLine {
        parser->line++;
    }

    action putChar {
        if (!string_add_checked(parser->string, *p))
            parser->finished = 1;
    }

    action startString {
        if (parser->string != NULL) {
            string_free(parser->string);
        }
        parser->string = string_new();
        if (parser->string == NULL) {
            parser->finished = 1;
        }
    }

    action endSimpleLiteral {
        emit(parser, ENTEE_SIMPLE_LITERAL_VALUE);
    }

    action datatypeLiteral {
        emit(parser, ENTEE_DATATYPE_LITERAL_VALUE);
    }

    action langtagLiteral {
        emit(parser, ENTEE_LANGUAGE_TAGGED_LITERAL_VALUE);
    }

    action endLangtagLiteral {
        emit(parser, ENTEE_LANGUAGE_TAGGED_LITERAL_LANGUAGE);
    }

    action endDatatypeLiteral {
        emit(parser, ENTEE_DATATYPE_LITERAL_IRI);
    }

    action endIRI {
        emit(parser, ENTEE_IRI);
    }

    action endBlankNode {
        emit(parser, ENTEE_BLANK_NODE);
    }

    WS = ' ' | '\t';
    EOL = '\r' | '\n';
    HEX = '0' .. '9' | 'A' .. 'F' | 'a' .. 'f';
    PN_CHARS_BASE = ('A' .. 'Z' | 'a' .. 'z'
                     # The N-Triple spec specifies Unicode codepoints,
                     # but files are encoded (and parsed) as UTF-8:
                     # 0x00C0 .. 0x00D6
                     | 0xC3 0x80..0x96
                     # 0x00D8 .. 0x00F6
                     | 0xC3 0x98..0xB6
                     # 0x00F8 .. 0x02FF
                     | 0xC3 0xB8..0xFF
                     | 0xC4..0xCA 0x00..0xFF
                     | 0xCB 0x00..0xBF
                     # 0x0370 .. 0x037D
                     | 0xCD 0xB0..0xBD
                     # 0x037F .. 0x1FFF = 0x037F .. 0x07FF | 0x0800 .. 0x1FFF
                     | 0xCD 0xBF..0xFF
                     | 0xCE..0xDE 0x00..0xFF
                     | 0xDF 0x00..0xBF
                     | 0xE0 0xA0 0x80..0xFF
                     | 0xE0 0xA1..0xFF 0x00..0xFF
                     | 0xE1 0x00 0x00..0xFF
                     | 0xE1 0x01..0xBE 0x00..0xFF
                     | 0xE1 0xBF 0x00..0xBF
                     # 0x200C .. 0x200D
                     | 0xE2 0x80 0x8C..0x8D
                     # 0x2070 .. 0x218F
                     | 0xE2 0x81 0xB0..0xFF
                     | 0xE2 0x82..0x85 0x00..0xFF
                     | 0xE2 0x86 0x00..0x8F
                     # 0x2C00 .. 0x2FEF
                     | 0xE2 0xB0 0x80..0xFF
                     | 0xE2 0xB1..0xBE 0x00..0xFF
                     | 0xE2 0xBF 0x00..0xAF
                     # 0x3001 .. 0xD7FF
                     | 0xE3 0x80 0x81..0xFF
                     | 0xE3 0x81..0xFF 0x00..0xFF
                     | 0xE4..0xEC 0x00..0xFF 0x00..0xFF
                     | 0xED 0x00 0x00..0xFF
                     | 0xED 0x01..0x9E 0x00..0xFF
                     | 0xED 0x9F 0x00..0xBF
                     # 0xF900 .. 0xFDCF
                     | 0xEF 0xA4 0x80..0xFF
                     | 0xEF 0xA5..0xB6 0x00..0xFF
                     | 0xEF 0xB7 0x00..0x8F
                     # 0xFDF0 .. 0xFFFD
                     | 0xEF 0xB7 0xB0..0xFF
                     | 0xEF 0xB8..0xBE 0x00..0xFF
                     | 0xEF 0xBF 0x00..0xBD
                     # 0x10000 .. 0xEFFFF
                     | 0xF0 0x90 0x80 0x80..0xFF
                     | 0xF0 0x90 0x81..0xFF 0x00..0xFF
                     | 0xF0 0x91..0xFF 0x00..0xFF 0x00..0xFF
                     | 0xF1..0xF2 0x00..0xFF 0x00..0xFF 0x00..0xFF
                     | 0xF3 0x00 0x00 0x00..0xFF
                     | 0xF3 0x00 0x01..0xFF 0x00..0xFF
                     | 0xF3 0x01..0xAE 0x00..0xFF 0x00..0xFF
                     | 0xF3 0xAF 0x00 0x00..0xFF
                     | 0xF3 0xAF 0x01..0xBE 0x00..0xFF
                     | 0xF3 0xAF 0xBF 0x00..0xBF
                     ) $putChar;
    PN_CHARS_U = PN_CHARS_BASE | (('_' | ':') $putChar);
    PN_CHARS = PN_CHARS_U
               | (('-' | '0' .. '9' | 0x00B7
                   # 0x0300 .. 0x036F
                   | 0xCC 0x80..0xFF
                   | 0xCD 0x00..0xAF
                   # 0x203F .. 0x2040
                   | 0xE2 0x80 0xBF..0xFF
                   | 0xE2 0x81 0x00..0x80)
                  $putChar);
    ECHAR = '\\' .
            (  ('t' ${ string_add_checked(parser->string, '\t'); })
             | ('b' ${ string_add_checked(parser->string, '\b'); })
             | ('n' ${ string_add_checked(parser->string, '\n'); })
             | ('r' ${ string_add_checked(parser->string, '\r'); })
             | ('f' ${ string_add_checked(parser->string, '\f'); })
             | ('"' ${ string_add_checked(parser->string, '\"'); })
             | ('\'' ${ string_add_checked(parser->string, '\''); })
             | ('\\' ${ string_add_checked(parser->string, '\\'); })
             );
    UCHAR = '\\' .
            (  ('u' HEX{4} >mark %unescape)
             | ('U' HEX{8} >mark %unescape));
    BLANK_NODE_LABEL = '_:' >startString (PN_CHARS_U | '0' .. '9' $putChar)
                       ((PN_CHARS | '.' $putChar)* PN_CHARS?);
    STRING_VALUE = (^(0x22 | 0x5C | 0xA | 0xD) $putChar | ECHAR | UCHAR)* >startString;
    IRIREF_VALUE = (^(0x00 .. 0x20 | '<' | '>' | '"' | '{' | '}' | '|' | '^' | '`' | '\\') $putChar | UCHAR)*
                   >startString;
    IRIREF = '<' IRIREF_VALUE '>';
    LANGTAG_VALUE = (('a' .. 'z' | 'A' .. 'Z') $putChar)+ >startString
                    ('-' $putChar (('a' .. 'z' | 'A' .. 'Z' | '0' .. '9') $putChar)+)*;
    datatypeLiteral = '"' STRING_VALUE '"' '^^' >datatypeLiteral IRIREF %endDatatypeLiteral;
    langtagLiteral =  '"' STRING_VALUE '"' '@' >langtagLiteral LANGTAG_VALUE %endLangtagLiteral;
    simpleLiteral = '"' STRING_VALUE '"' %endSimpleLiteral;
    literal = datatypeLiteral | langtagLiteral | simpleLiteral;
    subject = IRIREF %endIRI
            | BLANK_NODE_LABEL %endBlankNode;
    predicate = IRIREF %endIRI;
    object = literal
           | IRIREF %endIRI
           | BLANK_NODE_LABEL %endBlankNode;
    comment = '#' (any - EOL)*;
    # TODO: ensure left-guarded concatenation is correctly preventing
    #       ambiguity of blank nodes with dots
    triple = subject WS* predicate WS* object WS* <: '.' WS* %endTriple;
    line = WS* ( comment | triple )? (EOL %endLine)+;
    gobble_line := (any -- EOL)* EOL @{ fcall main; };
    main := line* $err{
        // failed on line arser->line
        fhold; fgoto gobble_line;
    };

    write data;
}%%

entee_parser *entee_new_parser() {
    entee_parser *parser = malloc(sizeof(entee_parser));
    parser->string = NULL;
    parser->line = 1;
    parser->finished = 0;
    parser->reader = NULL;
    parser->reader_arg = NULL;
    parser->handler = NULL;
    parser->handler_arg = NULL;
    return parser;
}

void entee_free_parser(entee_parser *parser) {
    free(parser);
}

void entee_parser_set_reader(entee_parser *parser,
                             entee_reader reader,
                             void *reader_arg)
{
    parser->reader = reader;
    parser->reader_arg = reader_arg;
}

void entee_parser_set_handler(entee_parser *parser,
                              entee_handler handler,
                              void *handler_arg)
{
    parser->handler = handler;
    parser->handler_arg = handler_arg;
}

int entee_parser_parse(entee_parser *parser) {
    if (parser->reader == NULL
        || parser->handler == NULL)
    {
        return 0;
    }

    int res = 1;
    int have = 0;
    int cs;
    int top;
    int stack[32];
    unsigned char buf[BUFSIZE];

    unsigned char *mark = 0;

    %% write init;

    entee_reader reader = parser->reader;

    while (!parser->finished) {
        unsigned char *p = buf + have;
        unsigned char *pe;
        unsigned char *eof = 0;

        int space = BUFSIZE - have;
        if (space == 0) {
            // out of buffer space
            res = 0;
            break;
        }

        int len = reader(p, space, parser->reader_arg);
        pe = p + len;

        if (len < space) {
            eof = pe;
            parser->finished = 1;
        }

        %% write exec;

        if (cs == ntriples_error) {
            res = 0;
            break;
        }

        if (mark == 0) {
            have = 0;
        } else {
            have = pe - mark;
            memmove(buf, mark, have);
            mark = buf;
        }
    }

    return res;
}


[confscanner] removed out-dated TODO

[confscanner] removed out-dated TODO

#include "config.h"
#include "confscanner.h"

#ifdef WIN32
	#include <windows.h>
	#include <shlwapi.h>
	#pragma comment (lib, "shlwapi.lib")
#else
	extern "C" {
		#include <fnmatch.h>
	}
#endif

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      //cerr << "glob matched!" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "adding: " << tmp_string.substr(0, found-1) + '\n';
          //cerr << endl;
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          /*if(!param_list_["grid"].empty())
            cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];*/
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
/*    cerr << "register glob: " << tmp_string << endl;
    cerr << " on top of globlist_stack " << endl;*/
    //cerr_stack();
//    cerr << "\n";
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{cblock_ = 0; fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                {
                  if(cblock_ != 0)
                    (*cblock_).clear();
                }
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list:\n"; cerr.write(p, 10); cerr << endl;*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2\n"; cerr.write(p, 20); cerr << endl;*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (MIDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok )
              . (MWSOC*
                 . (GLOB)
                )*
              . MWSOC*
              . ';'
                @{
                  arglist_ = &vars_[tmp_string];
                  arglist_->push_back(std::string(tmp_p, p - tmp_p));
                 }
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string is matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    const char * glob = (*it).c_str();
#ifndef WIN32
    int flags = 0;
#endif
    if (glob[0] == '.' && glob[1] == '/')
    {
      glob = &glob[2];
#ifndef WIN32
      flags = FNM_PATHNAME;
#endif
    }

    // check wether str matches glob
#ifdef WIN32
    if(PathMatchSpec(str,glob))
#else
    if(fnmatch(glob, str, flags) == 0)
#endif
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename is matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
  {
    //cerr << "matched " << s << endl;
    return true;
  }

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
    {
      //cerr << "matched " << s << endl;
      return true;
    }

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
    {
      //cerr << "matched " << s << endl;
      return true;
    }
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

/**
 * @class ConfFileScanner
 *
 * @change{1,2,dw,2011-11-07} Giving correct error message when no configuration file is given or found.
 */
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : buf(new char[BUFSIZE]), line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "./mtocpp.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  //cerr << "filename" << filename << endl;
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "mtoc++ No configuration file specified or file not found. Using default values." << endl;
    conffile_ = "";
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  if ( ! conffile_.empty() )
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    /* Do the first read. */
    bool done = false;
    while ( !done )
    {
      char *p = buf + have;
      char *tmp_p = p;// *tmp_p2 = p;
      // string for temporary tokens
      string tmp_string;
      // spare space in buffer
      int space = BUFSIZE - have;

      if ( space == 0 )
      {
        /* We filled up the buffer trying to scan a token. */
        cerr << "OUT OF BUFFER SPACE" << endl;
        exit(1);
      }

      // read configuration file chunk in buffer
      confistream_.read( p, space );
      int len = confistream_.gcount();
      char *pe = p + len;
      char *rpe = pe;

      /* If we see eof then append the EOF char. */
      if ( confistream_.eof() )
      {
        done = true;
      }
      else
      {
        /* Find the last semicolon by searching backwards. This
         * is where we will stop processing on this iteration. */
        while ( *pe!= ';' && pe > p )
          pe--;
      }

      //std::cerr << "execute parser" << std::endl;
      %% write exec;
      //std::cerr << "finish parser" << std::endl;

      /* Check if we failed. */
      if ( cs == ConfFileScanner_error )
      {
        /* Machine failed before finding a token. */
        cerr << "PARSE ERROR in mtoc++ config file, line " << line << endl;
        exit(1);
      }

      have = rpe - pe;
      /* cerr << "memmove by " << have << "bytes\n";*/
      memmove( buf, pe, have );
    }
    //cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "config.h"
#include "confscanner.h"

#ifdef WIN32
	#include <windows.h>
	#include <shlwapi.h>
	#pragma comment (lib, "shlwapi.lib")
#else
	extern "C" {
		#include <fnmatch.h>
	}
#endif

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      //cerr << "glob matched!" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "adding: " << tmp_string.substr(0, found-1) + '\n';
          //cerr << endl;
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          /*if(!param_list_["grid"].empty())
            cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];*/
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
/*    cerr << "register glob: " << tmp_string << endl;
    cerr << " on top of globlist_stack " << endl;*/
    //cerr_stack();
//    cerr << "\n";
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{cblock_ = 0; fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                {
                  if(cblock_ != 0)
                    (*cblock_).clear();
                }
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list:\n"; cerr.write(p, 10); cerr << endl;*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2\n"; cerr.write(p, 20); cerr << endl;*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (MIDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok )
              . (MWSOC*
                 . (GLOB)
                )*
              . MWSOC*
              . ';'
                @{
                  arglist_ = &vars_[tmp_string];
                  arglist_->push_back(std::string(tmp_p, p - tmp_p));
                 }
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string is matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    const char * glob = (*it).c_str();
#ifndef WIN32
    int flags = 0;
#endif
    if (glob[0] == '.' && glob[1] == '/')
    {
      glob = &glob[2];
#ifndef WIN32
      flags = FNM_PATHNAME;
#endif
    }

    // check wether str matches glob
#ifdef WIN32
    if(PathMatchSpec(str,glob))
#else
    if(fnmatch(glob, str, flags) == 0)
#endif
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename is matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
  {
    //cerr << "matched " << s << endl;
    return true;
  }

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
    {
      //cerr << "matched " << s << endl;
      return true;
    }

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
    {
      //cerr << "matched " << s << endl;
      return true;
    }
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

/**
 * @class ConfFileScanner
 *
 * @change{1,2,dw,2011-11-07} Giving correct error message when no configuration file is given or found.
 */
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : buf(new char[BUFSIZE]), line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "./mtocpp.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  //cerr << "filename" << filename << endl;
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "mtoc++ No configuration file specified or file not found. Using default values." << endl;
    conffile_ = "";
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  if ( ! conffile_.empty() )
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    /* Do the first read. */
    bool done = false;
    while ( !done )
    {
      char *p = buf + have;
      char *tmp_p = p;// *tmp_p2 = p;
      // string for temporary tokens
      string tmp_string;
      // spare space in buffer
      int space = BUFSIZE - have;

      if ( space == 0 )
      {
        /* We filled up the buffer trying to scan a token. */
        cerr << "OUT OF BUFFER SPACE" << endl;
        exit(1);
      }

      // read configuration file chunk in buffer
      confistream_.read( p, space );
      int len = confistream_.gcount();
      char *pe = p + len;
      char *rpe = pe;

      /* If we see eof then append the EOF char. */
      if ( confistream_.eof() )
      {
        done = true;
      }
      else
      {
        /* Find the last semicolon by searching backwards. This
         * is where we will stop processing on this iteration. */
        while ( *pe!= ';' && pe > p )
          pe--;
      }

      //std::cerr << "execute parser" << std::endl;
      %% write exec;
      //std::cerr << "finish parser" << std::endl;

      /* Check if we failed. */
      if ( cs == ConfFileScanner_error )
      {
        /* Machine failed before finding a token. */
        cerr << "PARSE ERROR in mtoc++ config file, line " << line << endl;
        exit(1);
      }

      have = rpe - pe;
      /* cerr << "memmove by " << have << "bytes\n";*/
      memmove( buf, pe, have );
    }
    //cerr << "afterwards param_list_[\"grid\"][0] " << param_list_["grid"][0];
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */


this one can go

this one can go

func rdata_a(hdr RR_Header, tok *token) RR {
    rr := new(RR_A)
    rr.Hdr = hdr
    rr.Hdr.Rrtype = TypeA
    rr.A = net.ParseIP(tok.T[0])
}
func rdata_ns(hdr RR_Header, tok *token) RR {
    rr := new(RR_NS)
    rr.Hdr = hdr
    rr.Hdr.Rrtype = TypeNS
    rr.Ns = tok.T[0]
}
func rdata_cname(hdr RR_Header, tok *token) RR {
    rr := new(RR_CNAME)
    rr.Hdr = hdr
    rr.Hdr.Rrtype = TypeCNAME
    rr.Cname = tok.T[0]
}
func rdata_soa(hdr RR_Header, tok *token) RR {
    rr := new(RR_SOA)
    rr.Hdr = hdr
    rr.Hdr.Rrtype = TypeSOA
    rr.Ns = tok.T[0]
    rr.Mbox = tok.T[1]
    rr.Serial = uint32(tok.N[0])
    rr.Refresh = uint32(tok.N[1])
    rr.Retry = uint32(tok.N[2])
    rr.Expire = uint32(tok.N[3])
    rr.Minttl = uint32(tok.N[4])
}
func rdata_mx(hdr RR_Header, tok *token) RR {
    rr := new(RR_MX)
    rr.Hdr = hdr;
    rr.Hdr.Rrtype = TypeMX
    rr.Pref = uint16(tok.N[0])
    rr.Mx = tok.T[0]
}
func rdata_ds(hdr RR_Header, tok *token) RR {
    rr := new(RR_DS)
    rr.Hdr = hdr;
    rr.Hdr.Rrtype = TypeDS
    rr.KeyTag = uint16(tok.N[0])
    rr.Algorithm = uint8(tok.N[1])
    rr.DigestType = uint8(tok.N[2])
    rr.Digest = tok.T[0]
}
func rdata_dnskey(hdr RR_Header, tok *token) RR {
    rr := new(RR_DNSKEY)
    rr.Hdr = hdr;
    rr.Hdr.Rrtype = TypeDNSKEY
    rr.Flags = uint16(tok.N[0])
    rr.Protocol = uint8(tok.N[1])
    rr.Algorithm = uint8(tok.N[2])
    rr.PublicKey = tok.T[0]
}
func rdata_rrsig(hdr RR_Header, tok *token) RR {
    rr := new(RR_RRSIG)
    rr.Hdr = hdr;
    rr.Hdr.Rrtype = TypeRRSIG
    rr.TypeCovered = uint16(tok.N[0])
    rr.Algorithm = uint8(tok.N[1])
    rr.Labels = uint8(tok.N[2])
    rr.OrigTtl = uint32(tok.N[3])
    rr.Expiration = uint32(tok.N[4])
    rr.Inception = uint32(tok.N[5])
    rr.KeyTag = uint16(tok.N[6])
    rr.SignerName = tok.T[0]
    rr.Signature = tok.T[1]
}


Add semicolons to emit statements to be more consistent

* The purpose of the formatting is that this code should work
  with C, Java and Ruby code.

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'   @{ emit('select'); };
    'FROM'     @{ emit('from');   };
    'AS'       @{ emit('as');     };

    # Literals
    'TRUE'     @{ emit('true');  };
    'FALSE'    @{ emit('false'); };
    'NULL'     @{ emit('null');  };

    # Operators
    '('        @{ emit('left_paren');  };
    ')'        @{ emit('right_paren'); };
    ','        @{ emit('comma');       };
    '.'        @{ emit('period');      };
    '+'        @{ emit('plus_sign');   };
    '-'        @{ emit('minus_sign');  };
    '*'        @{ emit('asterisk');    };
    '/'        @{ emit('solidus');     };
    'E'        @{ emit('E');           };

    digit      @{ emit('digit');      };
    identifier @{ emit('identifier'); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block

      # Tokenize a stream of input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def emit(token)
      @block.call(token.to_sym, text)
      @ts = nil
    end

    def text
      @buffer[@ts..@p].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'   @{ emit('select') };
    'FROM'     @{ emit('from')   };
    'AS'       @{ emit('as')     };

    # Literals
    'TRUE'     @{ emit('true');  };
    'FALSE'    @{ emit('false'); };
    'NULL'     @{ emit('null');  };

    # Operators
    '('        @{ emit('left_paren');  };
    ')'        @{ emit('right_paren'); };
    ','        @{ emit('comma');       };
    '.'        @{ emit('period');      };
    '+'        @{ emit('plus_sign');   };
    '-'        @{ emit('minus_sign');  };
    '*'        @{ emit('asterisk');    };
    '/'        @{ emit('solidus');     };
    'E'        @{ emit('E');           };

    digit      @{ emit('digit')       };
    identifier @{ emit('identifier'); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block

      # Tokenize a stream of input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def emit(token)
      @block.call(token.to_sym, text)
      @ts = nil
    end

    def text
      @buffer[@ts..@p].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL


hext-machine: fix comment

hext-machine: fix comment

%%{
# Copyright 2015 Thomas Trapp
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This ragel machine parses/lexes hext. It is embedded in ragel/Parser.cpp.rl.
machine hext;

#### HELPERS ###################################################################
# Set an attribute capture to optional. By default, attribute captures are
# mandatory, i.e. the attribute has to exist in a node for it to match.
optional = (
  '?' %{ pv.optional = true; }
);
# Negate a match pattern, e.g. style!, class="menu"!, @text=~/foo/!
negate = (
  '!' %{ pv.set_test<NegateTest>(std::move(pv.test)); }
);
# The name of an HTML-element's attribute
attr_name = (
  ( alpha (alnum | '-' | '_')** )
  >{ tk_start(); }
  %{ tk_stop(); pv.attr_name = tok; }
);
# Quoted literal value that may be empty, i.e. "value" or 'value'
opt_quoted =
( '"' (
    ( [^"] | '\\"' )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( [^'] | '\\\'' )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '\'' );
# Quoted literal value that may not be empty, i.e. "value" or 'value'
quoted =
( '"' (
    ( ([^"] | '\\"')+ )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( ([^'] | '\\\'')+ )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '\'' );

#### NTH_PATTERN ###############################################################
# nth_pattern is used by traits, e.g. nth-last-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  ( 'even' %{ pv.nth = {2, 0}; } )
  |

  ( 'odd' %{ pv.nth = {2, 1}; } )
  |

  ( ( ( '-'? [0-9]+ )
      >{ tk_start(); }
      %{ tk_stop(); pv.nth = {0, std::stoi(tok)}; }
    )
    ( 'n'
      %{ pv.nth = {pv.nth.second, 0}; }
      ( ( ('+'|'-') [0-9]+ )
        >{ tk_start(); }
        %{ tk_stop(); pv.nth.second = std::stoi(tok); }
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  ( 'empty' %{ pv.set_trait<ChildCountMatch>(0); } )
  |

  ( 'child-count('
    ( [0-9]+
      >{ tk_start(); }
      %{ tk_stop(); pv.set_trait<ChildCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'attribute-count('
    ( [0-9]+
      >{ tk_start(); }
      %{ tk_stop(); pv.set_trait<AttributeCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'nth-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth); } )
  |

  ( 'nth-last-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last); } )
  |

  ( 'nth-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'first-child'
    %{ pv.set_trait<NthChildMatch>(0, 1); } )
  |

  ( 'first-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'last-child'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last); } )
  |

  ( 'last-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'only-child' %{ pv.set_trait<OnlyChildMatch>(); } )
  |

  ( 'only-of-type' %{ pv.set_trait<OnlyChildMatch>(OnlyChildMatch::OfType); } )
);
not_trait =
':not('
  %{ pv.negate = std::make_unique<NegateMatch>(); }
  (
    trait
    %{
       assert(pv.negate);
       pv.negate->append_match(std::move(pv.trait));
     }
  )+
')';

#### REGULAR EXPRESSIONS #######################################################
regex =
'/' ( ( [^/] | '\\/' )** >{ tk_start(); } %{ tk_stop(); } ) '/'
# regex flags
(
  # case insensitive
  ( 'i' %{ pv.regex_flag |= boost::regex::icase; } )
  |
  # collate (locale aware character groups)
  ( 'c' %{ pv.regex_flag |= boost::regex::collate; } )
)*
%{ try {
     pv.regex = boost::regex(tok, pv.regex_flag);
   }
   catch( const boost::regex_error& e ) {
     // Mark whole regex as error, including slashes and flags
     auto mark_len = static_cast<std::size_t>(this->p - tok_begin + 1);
     this->throw_regex_error(mark_len, e.code());
   }
};
# Wrap a regular expression into a RegexTest.
regex_test = regex %{ assert(pv.regex); pv.set_test<RegexTest>(*pv.regex); };

#### BUILTIN FUNCTION ##########################################################
builtin = (
  '@'
  ( ( 'text'       %{ pv.builtin = TextBuiltin; } )
    |
    ( 'inner-html' %{ pv.builtin = InnerHtmlBuiltin; } )
    |
    ( 'strip-tags' %{ pv.builtin = StripTagsBuiltin; } ) )
);

#### CAPTURE ###################################################################
# captures can be passed through StringPipes, e.g. id:trim:id
pipe = (
  ':'
  ( ( ( 'trim' )                %{ pv.add_pipe<TrimPipe>(); } )
    |
    ( ( 'trim(' quoted ')' )    %{ pv.add_pipe<TrimPipe>(pv.literal_value); } )
    |
    ( ( 'tolower' )             %{ pv.add_pipe<CasePipe>(); } )
    |
    ( ( 'toupper' )             %{ pv.add_pipe<CasePipe>(CasePipe::ToUpper); } )
    |
    ( ( 'prepend(' quoted ')' ) %{ pv.add_pipe<PrependPipe>(pv.literal_value); } )
    |
    ( ( 'append(' quoted ')' )  %{ pv.add_pipe<AppendPipe>(pv.literal_value); } )
    |
    ( ( 'filter(' regex ')' )   %{ pv.add_pipe<RegexPipe>(*pv.regex); } )
    |
    ( ( 'replace(' regex space* ',' space* opt_quoted ')' )
        %{ pv.add_pipe<RegexReplacePipe>(*pv.regex, pv.literal_value); } ) )
);
# capture variable, e.g. id:linkid, id:trim:linkid, id:"Menu ID"
capture = (
  pipe* ':'
  (
    ( quoted %{ pv.cap_var = pv.literal_value; } )
    |
    ( (alnum | [\-_.])+ >{ tk_start(); }
                        %{ tk_stop(); pv.cap_var = tok; } ) )
);

#### MATCH_ARG #################################################################
match_arg = (
  ( '=' quoted
    %{ pv.set_test<ContainsWordsTest>(pv.literal_value); } )
  |
  ( '==' opt_quoted
    %{ pv.set_test<EqualsTest>(pv.literal_value); } )
  |
  ( '^=' quoted
    %{ pv.set_test<BeginsWithTest>(pv.literal_value); } )
  |
  ( '*=' quoted
    %{ pv.set_test<ContainsTest>(pv.literal_value); } )
  |
  ( '$=' quoted
    %{ pv.set_test<EndsWithTest>(pv.literal_value); } )
);

#### PATTERNS ##################################################################
pattern = (
  space+
  ( ( ( builtin capture )
      %{ cur_rule().append_capture<FunctionCapture>(
             pv.builtin, pv.cap_var, std::move(pv.pipe)); } )
    |

    ( ( builtin '=~' regex_test negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( builtin match_arg negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( attr_name capture optional? )
      %{ cur_rule().append_capture<AttributeCapture>(
             pv.attr_name, pv.cap_var, std::move(pv.pipe));
         if( !pv.optional )
           cur_rule().append_match<AttributeMatch>(pv.attr_name);
       } )
    |

    ( ( attr_name '=~' regex_test negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name match_arg negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name
        %{ pv.test = nullptr; }
        negate?
      )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
  ) %{ pv.reset(); }
);

#### RULES #####################################################################
tag_name = ( alpha (alnum | '-' | '_')** );
main := (
  # ignore whitespace
  space
  |

  # comment
  ( '#' (any - '\n')* '\n' )
  |

  # open rule
  (
    # a rule starts with '<'
    ( '<' %{ push_rule(); } )

    # a rule can be optional
    ( '?' %{ cur_rule().set_optional(true); } )?

    # a rule must have a tag name, e.g. <div
    ( ( '*' | tag_name ) >{ tk_start(); }
                         %{ tk_stop(); set_open_tag_or_throw(tok); } )

    # a rule can have multiple traits, e.g. :first-child, :empty
    ( ( not_trait %{ cur_rule().append_match(std::move(pv.negate)); } )
      |

      ( trait %{ cur_rule().append_match(std::move(pv.trait)); } ) )*

    # a rule can have multiple match or capture patterns,
    # e.g. class="menu", @text:heading
    pattern*

    space*

    ( ( '/>' %{ pop_rule(); } ) | '>' )
  )
  |

  # end rule
  (
    '</'
    ( ( '*' | tag_name ) >{ tk_start(); }
                         %{ tk_stop();
                            validate_close_tag_or_throw(tok);
                            pop_rule(); } )
    '>'
  )
)* $err{ this->throw_unexpected(); };

}%%

%%{
# Copyright 2015 Thomas Trapp
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This ragel machine parses/lexes hext. It is embedded in ragel/Parser.cpp.rl.
machine hext;

#### HELPERS ###################################################################
# Set an attribute capture to optional. By default, attribute captures are
# mandatory, i.e. the attribute has to exist in a node for it to match.
optional = (
  '?' %{ pv.optional = true; }
);
# Negate a match pattern, e.g. style!, class="menu"!, @text=~/foo/!
negate = (
  '!' %{ pv.set_test<NegateTest>(std::move(pv.test)); }
);
# The name of an HTML-element's attribute
attr_name = (
  ( alpha (alnum | '-' | '_')** )
  >{ tk_start(); }
  %{ tk_stop(); pv.attr_name = tok; }
);
# Quoted literal value that may be empty, i.e. "value" or 'value'
opt_quoted =
( '"' (
    ( [^"] | '\\"' )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( [^'] | '\\\'' )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '\'' );
# Quoted literal value that may not be empty, i.e. "value" or 'value'
quoted =
( '"' (
    ( ([^"] | '\\"')+ )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '"' )
|
( '\'' (
    ( ([^'] | '\\\'')+ )**
    >{ tk_start(); } %{ tk_stop(); pv.literal_value = tok; }
  ) '\'' );

#### NTH_PATTERN ###############################################################
# nth_pattern is used by traits, e.g. nth-last-child(nth_pattern).
# Examples: :nth-child(1)  :nth-child(even)  :nth-child(2n)  :nth-child(2n+1)
nth_pattern = (
  ( 'even' %{ pv.nth = {2, 0}; } )
  |

  ( 'odd' %{ pv.nth = {2, 1}; } )
  |

  ( ( ( '-'? [0-9]+ )
      >{ tk_start(); }
      %{ tk_stop(); pv.nth = {0, std::stoi(tok)}; }
    )
    ( 'n'
      %{ pv.nth = {pv.nth.second, 0}; }
      ( ( ('+'|'-') [0-9]+ )
        >{ tk_start(); }
        %{ tk_stop(); pv.nth.second = std::stoi(tok); }
      )?
    )?
  )
);

#### TRAITS ####################################################################
trait = ':' (
  ( 'empty' %{ pv.set_trait<ChildCountMatch>(0); } )
  |

  ( 'child-count('
    ( [0-9]+
      >{ tk_start(); }
      %{ tk_stop(); pv.set_trait<ChildCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'attribute-count('
    ( [0-9]+
      >{ tk_start(); }
      %{ tk_stop(); pv.set_trait<AttributeCountMatch>(std::stoi(tok)); } )
    ')' )
  |

  ( 'nth-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth); } )
  |

  ( 'nth-last-child(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last); } )
  |

  ( 'nth-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'first-child'
    %{ pv.set_trait<NthChildMatch>(0, 1); } )
  |

  ( 'first-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::First|NthChildMatch::OfType); } )
  |

  ( 'last-child'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last); } )
  |

  ( 'last-of-type'
    %{ pv.set_trait<NthChildMatch>(0, 1, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'nth-last-of-type(' nth_pattern ')'
    %{ pv.set_trait<NthChildMatch>(pv.nth, NthChildMatch::Last|NthChildMatch::OfType); } )
  |

  ( 'only-child' %{ pv.set_trait<OnlyChildMatch>(); } )
  |

  ( 'only-of-type' %{ pv.set_trait<OnlyChildMatch>(OnlyChildMatch::OfType); } )
);
not_trait =
':not('
  %{ pv.negate = std::make_unique<NegateMatch>(); }
  (
    trait
    %{
       assert(pv.negate);
       pv.negate->append_match(std::move(pv.trait));
     }
  )+
')';

#### REGULAR EXPRESSIONS #######################################################
regex =
'/' ( ( [^/] | '\\/' )** >{ tk_start(); } %{ tk_stop(); } ) '/'
# regex flags
(
  # case insensitive
  ( 'i' %{ pv.regex_flag |= boost::regex::icase; } )
  |
  # collate (locale aware character groups)
  ( 'c' %{ pv.regex_flag |= boost::regex::collate; } )
)*
%{ try {
     pv.regex = boost::regex(tok, pv.regex_flag);
   }
   catch( const boost::regex_error& e ) {
     // Mark whole regex as error, including slashes and flags
     auto mark_len = static_cast<std::size_t>(this->p - tok_begin + 1);
     this->throw_regex_error(mark_len, e.code());
   }
};
# Wrap a regular expression into a RegexTest.
regex_test = regex %{ assert(pv.regex); pv.set_test<RegexTest>(*pv.regex); };

#### BUILTIN FUNCTION ##########################################################
builtin = (
  '@'
  ( ( 'text'       %{ pv.builtin = TextBuiltin; } )
    |
    ( 'inner-html' %{ pv.builtin = InnerHtmlBuiltin; } )
    |
    ( 'strip-tags' %{ pv.builtin = StripTagsBuiltin; } ) )
);

#### CAPTURE ###################################################################
# captures can be passed through StringPipes, e.g. id:trim->id
pipe = (
  ':'
  ( ( ( 'trim' )                %{ pv.add_pipe<TrimPipe>(); } )
    |
    ( ( 'trim(' quoted ')' )    %{ pv.add_pipe<TrimPipe>(pv.literal_value); } )
    |
    ( ( 'tolower' )             %{ pv.add_pipe<CasePipe>(); } )
    |
    ( ( 'toupper' )             %{ pv.add_pipe<CasePipe>(CasePipe::ToUpper); } )
    |
    ( ( 'prepend(' quoted ')' ) %{ pv.add_pipe<PrependPipe>(pv.literal_value); } )
    |
    ( ( 'append(' quoted ')' )  %{ pv.add_pipe<AppendPipe>(pv.literal_value); } )
    |
    ( ( 'filter(' regex ')' )   %{ pv.add_pipe<RegexPipe>(*pv.regex); } )
    |
    ( ( 'replace(' regex space* ',' space* opt_quoted ')' )
        %{ pv.add_pipe<RegexReplacePipe>(*pv.regex, pv.literal_value); } ) )
);
# capture variable, e.g. id:linkid, id:trim:linkid, id:"Menu ID"
capture = (
  pipe* ':'
  (
    ( quoted %{ pv.cap_var = pv.literal_value; } )
    |
    ( (alnum | [\-_.])+ >{ tk_start(); }
                        %{ tk_stop(); pv.cap_var = tok; } ) )
);

#### MATCH_ARG #################################################################
match_arg = (
  ( '=' quoted
    %{ pv.set_test<ContainsWordsTest>(pv.literal_value); } )
  |
  ( '==' opt_quoted
    %{ pv.set_test<EqualsTest>(pv.literal_value); } )
  |
  ( '^=' quoted
    %{ pv.set_test<BeginsWithTest>(pv.literal_value); } )
  |
  ( '*=' quoted
    %{ pv.set_test<ContainsTest>(pv.literal_value); } )
  |
  ( '$=' quoted
    %{ pv.set_test<EndsWithTest>(pv.literal_value); } )
);

#### PATTERNS ##################################################################
pattern = (
  space+
  ( ( ( builtin capture )
      %{ cur_rule().append_capture<FunctionCapture>(
             pv.builtin, pv.cap_var, std::move(pv.pipe)); } )
    |

    ( ( builtin '=~' regex_test negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( builtin match_arg negate? )
      %{ cur_rule().append_match<FunctionValueMatch>(pv.builtin, std::move(pv.test)); } )
    |

    ( ( attr_name capture optional? )
      %{ cur_rule().append_capture<AttributeCapture>(
             pv.attr_name, pv.cap_var, std::move(pv.pipe));
         if( !pv.optional )
           cur_rule().append_match<AttributeMatch>(pv.attr_name);
       } )
    |

    ( ( attr_name '=~' regex_test negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name match_arg negate? )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
    |

    ( ( attr_name
        %{ pv.test = nullptr; }
        negate?
      )
      %{ cur_rule().append_match<AttributeMatch>(pv.attr_name, std::move(pv.test)); } )
  ) %{ pv.reset(); }
);

#### RULES #####################################################################
tag_name = ( alpha (alnum | '-' | '_')** );
main := (
  # ignore whitespace
  space
  |

  # comment
  ( '#' (any - '\n')* '\n' )
  |

  # open rule
  (
    # a rule starts with '<'
    ( '<' %{ push_rule(); } )

    # a rule can be optional
    ( '?' %{ cur_rule().set_optional(true); } )?

    # a rule must have a tag name, e.g. <div
    ( ( '*' | tag_name ) >{ tk_start(); }
                         %{ tk_stop(); set_open_tag_or_throw(tok); } )

    # a rule can have multiple traits, e.g. :first-child, :empty
    ( ( not_trait %{ cur_rule().append_match(std::move(pv.negate)); } )
      |

      ( trait %{ cur_rule().append_match(std::move(pv.trait)); } ) )*

    # a rule can have multiple match or capture patterns,
    # e.g. class="menu", @text:heading
    pattern*

    space*

    ( ( '/>' %{ pop_rule(); } ) | '>' )
  )
  |

  # end rule
  (
    '</'
    ( ( '*' | tag_name ) >{ tk_start(); }
                         %{ tk_stop();
                            validate_close_tag_or_throw(tok);
                            pop_rule(); } )
    '>'
  )
)* $err{ this->throw_unexpected(); };

}%%


Move the STORE_URL on link aliases to match the cleanup in [253].

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#define superredcloth_scan_c

#include <ruby.h>
#include "superredcloth.h"

VALUE super_ParseError, super_RedCloth, super_HTML;

%%{

  machine superredcloth_scan;
  include superredcloth_common "ext/superredcloth_scan/superredcloth_common.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  # blocks
  notextile_tag_start = "<notextile>" ;
  notextile_tag_end = "</notextile>" ;
  notextile_line = " " (( default+ ) -- CRLF) CRLF ;
  notextile_block_start = ( "notextile" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  pre_tag_start = "
]* ">" (space* "")? ;
  pre_tag_end = ("" space*)? "
" ;
  pre_block_start = ( "pre" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  bc_start = ( "bc" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  bq_start = ( "bq" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) ( ":" %A uri %{ STORE(cite) } )? " "+ ) ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "div" ) ;
  block_start = ( btype >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  next_block_start = ( btype A C :> "." ) ;
  double_return = CRLF{2,} ;
  block_end = ( double_return | EOF );
  extended_block_end = double_return . next_block_start >A @{ p = reg - 1; } ;
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  footnote_start = ( ftype A C :> dotspace ) ;
  ul = "*" %{nest++; list_type = "ul";};
  ol = "#" %{nest++; list_type = "ol";};
  list_start  = ( ( ul | ol )+ N A C :> " "+ ) >{nest = 0;} ;
  dl_start = "-" . " "+ ;
  dd_start = ":=" ;
  blank_line = CRLF;
  link_alias = ( "[" >{ ASET(type, ignore) } %A phrase %T "]" %A uri %{ STORE_URL(href); } ) ;

  # html blocks
  BlockTagName = Name* - ("pre" | "notextile" | "a" | "applet" | "basefont" | "bdo" | "br" | "font" | "iframe" | "img" | "map" | "object" | "param" | "q" | "script" | "span" | "sub" | "sup" | "abbr" | "acronym" | "cite" | "code" | "del" | "dfn" | "em" | "ins" | "kbd" | "samp" | "strong" | "var" | "b" | "big" | "i" | "s" | "small" | "strike" | "tt" | "u");
  block_start_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? ">" | "<" BlockTagName ">";
  block_empty_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? "/>" | "<" BlockTagName "/>" ;
  block_end_tag = "</" BlockTagName space* ">" ;
  html_start = indent (block_start_tag | block_empty_tag) indent ;
  html_end = indent block_end_tag indent CRLF* ;
  standalone_html = indent (block_start_tag | block_empty_tag | block_end_tag) indent CRLF+;

  # tables
  para = ( default+ ) -- CRLF ;
  btext = para ( CRLF{2} )? ;
  tddef = ( D? S A C :> dotspace ) ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, text, td);} ) >X ;
  trdef = ( A C :> dotspace ) ;
  tr = ( trdef? "|" %{INLINE(table, tr_open);} td+ ) >X %{INLINE(table, tr_close);} ;
  trows = ( tr (CRLF >X tr)* ) ;
  tdef = ( "table" >X A C :> dotspace CRLF ) ;
  table = ( tdef? trows >{INLINE(table, table_open);} ) >{ reg = NULL; } ;

  pre_tag := |*
    pre_tag_end         { CAT(block); DONE(block); fgoto main; };
    default => esc_pre;
  *|;

  pre_block := |*
    EOF                { ADD_BLOCKCODE(); fgoto main; };
    extended_block_end { ADD_BLOCKCODE(); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCKCODE(); fgoto main; } else { ADD_EXTENDED_BLOCKCODE(); } };
    default => esc_pre;
  *|;

  notextile_tag := |*
    notextile_tag_end   { DONE(block); fgoto main; };
    default => cat;
  *|;

  notextile_block := |*
    EOF                { DONE(block); fgoto main; };
    extended_block_end { DONE(block); fgoto main; };
    double_return      { if (NIL_P(extend)) { DONE(block); fgoto main; } else { DONE(block); } };
    default => cat;
  *|;

  html := |*
    html_end        { CAT(block); ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  bc := |*
    EOF                { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; };
    extended_block_end { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; } else { ADD_EXTENDED_BLOCKCODE(); } };
    default => esc_pre;
  *|;

  bq := |*
    EOF                { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; };
    extended_block_end { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; } else { ADD_EXTENDED_BLOCK(); } };
    default => cat;
  *|;

  block := |*
    EOF                { ADD_BLOCK(); fgoto main; };
    extended_block_end { ADD_BLOCK(); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCK(); fgoto main; } else { ADD_EXTENDED_BLOCK(); } };
    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  list := |*
    CRLF list_start { ADD_BLOCK(); LIST_ITEM(); };
    block_end       { ADD_BLOCK(); nest = 0; LIST_CLOSE(); fgoto main; };
    default => cat;
  *|;

  dl := |*
    CRLF dl_start   { ADD_BLOCK(); ASET(type, dt); };
    dd_start        { ADD_BLOCK(); ASET(type, dd); };
    block_end       { ADD_BLOCK(); INLINE(html, dl_close);  fgoto main; };
    default => cat;
  *|;

  main := |*
    notextile_line  { CAT(block); DONE(block); };
    notextile_tag_start { ASET(type, notextile); fgoto notextile_tag; };
    notextile_block_start { fgoto notextile_block; };
    pre_tag_start       { ASET(type, notextile); CAT(block); fgoto pre_tag; };
    pre_block_start { fgoto pre_block; };
    standalone_html { CAT(block); DONE(block); };
    html_start      { ASET(type, notextile); CAT(block); fgoto html; };
    bc_start        { INLINE(html, bc_open); ASET(type, code); plain_block = rb_str_new2("code"); fgoto bc; };
    bq_start        { INLINE(html, bq_open); ASET(type, p); fgoto bq; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    list_start      { list_layout = rb_ary_new(); LIST_ITEM(); fgoto list; };
    dl_start        { INLINE(html, dl_open); ASET(type, dt); fgoto dl; };
    table           { INLINE(table, table_close); DONE(table); fgoto block; };
    link_alias      { rb_hash_aset(refs_found, rb_hash_aref(regs, ID2SYM(rb_intern("text"))), rb_hash_aref(regs, ID2SYM(rb_intern("href")))); DONE(block); };
    blank_line => cat;
    default
    {
      regs = rb_hash_new();
      rb_hash_aset(regs, ID2SYM(rb_intern("type")), plain_block);
      CAT(block);
      fgoto block;
    };
    EOF;
  *|;

}%%

%% write data nofinal;

VALUE
superredcloth_transform(rb_formatter, p, pe, refs)
  VALUE rb_formatter;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest;
  char *ts = NULL, *te = NULL, *reg = NULL, *eof = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = rb_hash_new();
  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  int list_continue = 0;
  VALUE plain_block = rb_str_new2("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return superredcloth_transform(rb_formatter, orig_p, orig_pe, refs_found);
  } else {
    return html;
  }
}

VALUE
superredcloth_transform2(formatter, str)
  VALUE formatter, str;
{
  rb_str_cat2(str, "\n");
  StringValue(str);
  return superredcloth_transform(formatter, RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1, Qnil);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(super_HTML, self);
}

static VALUE
superredcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(formatter, self);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  rb_define_method(super_RedCloth, "to", superredcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  super_HTML = rb_define_module_under(super_RedCloth, "HTML");
}

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#define superredcloth_scan_c

#include <ruby.h>
#include "superredcloth.h"

VALUE super_ParseError, super_RedCloth, super_HTML;

%%{

  machine superredcloth_scan;
  include superredcloth_common "ext/superredcloth_scan/superredcloth_common.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  # blocks
  notextile_tag_start = "<notextile>" ;
  notextile_tag_end = "</notextile>" ;
  notextile_line = " " (( default+ ) -- CRLF) CRLF ;
  notextile_block_start = ( "notextile" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  pre_tag_start = "
]* ">" (space* "")? ;
  pre_tag_end = ("" space*)? "
" ;
  pre_block_start = ( "pre" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  bc_start = ( "bc" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  bq_start = ( "bq" >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) ( ":" %A uri %{ STORE(cite) } )? " "+ ) ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "div" ) ;
  block_start = ( btype >A %{ STORE(type) } A C :> "." ( "." %extend | "" ) " "+ ) ;
  next_block_start = ( btype A C :> "." ) ;
  double_return = CRLF{2,} ;
  block_end = ( double_return | EOF );
  extended_block_end = double_return . next_block_start >A @{ p = reg - 1; } ;
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  footnote_start = ( ftype A C :> dotspace ) ;
  ul = "*" %{nest++; list_type = "ul";};
  ol = "#" %{nest++; list_type = "ol";};
  list_start  = ( ( ul | ol )+ N A C :> " "+ ) >{nest = 0;} ;
  dl_start = "-" . " "+ ;
  dd_start = ":=" ;
  blank_line = CRLF;
  link_alias = ( "[" >{ ASET(type, ignore) } %A phrase %T "]" %A uri ) ;

  # html blocks
  BlockTagName = Name* - ("pre" | "notextile" | "a" | "applet" | "basefont" | "bdo" | "br" | "font" | "iframe" | "img" | "map" | "object" | "param" | "q" | "script" | "span" | "sub" | "sup" | "abbr" | "acronym" | "cite" | "code" | "del" | "dfn" | "em" | "ins" | "kbd" | "samp" | "strong" | "var" | "b" | "big" | "i" | "s" | "small" | "strike" | "tt" | "u");
  block_start_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? ">" | "<" BlockTagName ">";
  block_empty_tag = "<" BlockTagName space+ AttrSet* (AttrEnd)? "/>" | "<" BlockTagName "/>" ;
  block_end_tag = "</" BlockTagName space* ">" ;
  html_start = indent (block_start_tag | block_empty_tag) indent ;
  html_end = indent block_end_tag indent CRLF* ;
  standalone_html = indent (block_start_tag | block_empty_tag | block_end_tag) indent CRLF+;

  # tables
  para = ( default+ ) -- CRLF ;
  btext = para ( CRLF{2} )? ;
  tddef = ( D? S A C :> dotspace ) ;
  td = ( tddef? btext >A %T :> "|" >{PASS(table, text, td);} ) >X ;
  trdef = ( A C :> dotspace ) ;
  tr = ( trdef? "|" %{INLINE(table, tr_open);} td+ ) >X %{INLINE(table, tr_close);} ;
  trows = ( tr (CRLF >X tr)* ) ;
  tdef = ( "table" >X A C :> dotspace CRLF ) ;
  table = ( tdef? trows >{INLINE(table, table_open);} ) >{ reg = NULL; } ;

  pre_tag := |*
    pre_tag_end         { CAT(block); DONE(block); fgoto main; };
    default => esc_pre;
  *|;

  pre_block := |*
    EOF                { ADD_BLOCKCODE(); fgoto main; };
    extended_block_end { ADD_BLOCKCODE(); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCKCODE(); fgoto main; } else { ADD_EXTENDED_BLOCKCODE(); } };
    default => esc_pre;
  *|;

  notextile_tag := |*
    notextile_tag_end   { DONE(block); fgoto main; };
    default => cat;
  *|;

  notextile_block := |*
    EOF                { DONE(block); fgoto main; };
    extended_block_end { DONE(block); fgoto main; };
    double_return      { if (NIL_P(extend)) { DONE(block); fgoto main; } else { DONE(block); } };
    default => cat;
  *|;

  html := |*
    html_end        { CAT(block); ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  bc := |*
    EOF                { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; };
    extended_block_end { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCKCODE(); INLINE(html, bc_close); plain_block = rb_str_new2("p"); fgoto main; } else { ADD_EXTENDED_BLOCKCODE(); } };
    default => esc_pre;
  *|;

  bq := |*
    EOF                { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; };
    extended_block_end { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCK(); INLINE(html, bq_close); fgoto main; } else { ADD_EXTENDED_BLOCK(); } };
    default => cat;
  *|;

  block := |*
    EOF                { ADD_BLOCK(); fgoto main; };
    extended_block_end { ADD_BLOCK(); fgoto main; };
    double_return      { if (NIL_P(extend)) { ADD_BLOCK(); fgoto main; } else { ADD_EXTENDED_BLOCK(); } };
    default => cat;
  *|;

  footnote := |*
    block_end       { ADD_BLOCK(); fgoto main; };
    default => cat;
  *|;

  list := |*
    CRLF list_start { ADD_BLOCK(); LIST_ITEM(); };
    block_end       { ADD_BLOCK(); nest = 0; LIST_CLOSE(); fgoto main; };
    default => cat;
  *|;

  dl := |*
    CRLF dl_start   { ADD_BLOCK(); ASET(type, dt); };
    dd_start        { ADD_BLOCK(); ASET(type, dd); };
    block_end       { ADD_BLOCK(); INLINE(html, dl_close);  fgoto main; };
    default => cat;
  *|;

  main := |*
    notextile_line  { CAT(block); DONE(block); };
    notextile_tag_start { ASET(type, notextile); fgoto notextile_tag; };
    notextile_block_start { fgoto notextile_block; };
    pre_tag_start       { ASET(type, notextile); CAT(block); fgoto pre_tag; };
    pre_block_start { fgoto pre_block; };
    standalone_html { CAT(block); DONE(block); };
    html_start      { ASET(type, notextile); CAT(block); fgoto html; };
    bc_start        { INLINE(html, bc_open); ASET(type, code); plain_block = rb_str_new2("code"); fgoto bc; };
    bq_start        { INLINE(html, bq_open); ASET(type, p); fgoto bq; };
    block_start     { fgoto block; };
    footnote_start  { fgoto footnote; };
    list_start      { list_layout = rb_ary_new(); LIST_ITEM(); fgoto list; };
    dl_start        { INLINE(html, dl_open); ASET(type, dt); fgoto dl; };
    table           { INLINE(table, table_close); DONE(table); fgoto block; };
    link_alias      { STORE_URL(href); rb_hash_aset(refs_found, rb_hash_aref(regs, ID2SYM(rb_intern("text"))), rb_hash_aref(regs, ID2SYM(rb_intern("href")))); DONE(block); };
    blank_line => cat;
    default
    {
      regs = rb_hash_new();
      rb_hash_aset(regs, ID2SYM(rb_intern("type")), plain_block);
      CAT(block);
      fgoto block;
    };
    EOF;
  *|;

}%%

%% write data nofinal;

VALUE
superredcloth_transform(rb_formatter, p, pe, refs)
  VALUE rb_formatter;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest;
  char *ts = NULL, *te = NULL, *reg = NULL, *eof = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = rb_hash_new();
  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  int list_continue = 0;
  VALUE plain_block = rb_str_new2("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return superredcloth_transform(rb_formatter, orig_p, orig_pe, refs_found);
  } else {
    return html;
  }
}

VALUE
superredcloth_transform2(formatter, str)
  VALUE formatter, str;
{
  rb_str_cat2(str, "\n");
  StringValue(str);
  return superredcloth_transform(formatter, RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1, Qnil);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(super_HTML, self);
}

static VALUE
superredcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  return superredcloth_transform2(formatter, self);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  rb_define_method(super_RedCloth, "to", superredcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  super_HTML = rb_define_module_under(super_RedCloth, "HTML");
}