throw lex_error from state machine on syntax error

%%{
  machine hext;

  action error {
    print_error();
    throw lex_error("syntax error");
  }

  cap_limit = ( '*' | [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      (
        '  ' >{ LX_TK_START(TK_INDENT); }
             %{ LX_TK_STOP; }
      )*
      (
        '<' >{ LX_TK_START(TK_RULE_BEGIN); }
            %{ LX_TK_STOP; }
      )
      (
        '!' >{ LX_TK_START(TK_DIRECT_DESC); }
            %{ LX_TK_STOP; }
      )?
      (
        cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                  %{ LX_TK_STOP; }
      )?
      (
        attr_name >{ LX_TK_START(TK_TAG_NAME); }
                  %{ LX_TK_STOP; }
      )?
      attributes?
      (
        '>' >{ LX_TK_START(TK_RULE_END); }
            %{ LX_TK_STOP; }
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    print_error();
    fbreak;
  }

  cap_limit = ( '*' | [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      (
        '  ' >{ LX_TK_START(TK_INDENT); }
             %{ LX_TK_STOP; }
      )*
      (
        '<' >{ LX_TK_START(TK_RULE_BEGIN); }
            %{ LX_TK_STOP; }
      )
      (
        '!' >{ LX_TK_START(TK_DIRECT_DESC); }
            %{ LX_TK_STOP; }
      )?
      (
        cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                  %{ LX_TK_STOP; }
      )?
      (
        attr_name >{ LX_TK_START(TK_TAG_NAME); }
                  %{ LX_TK_STOP; }
      )?
      attributes?
      (
        '>' >{ LX_TK_START(TK_RULE_END); }
            %{ LX_TK_STOP; }
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%


Case-insensitive well-known header parsing.

// Copyright (c) 2009 - Decho Corp.
/* To compile to .cpp:
   ragel parser.rl -G2 -o parser.cpp
*/

#include "parser.h"

// From uri.rl
std::string unescape(const std::string& str);

static
HTTP::Version
parseVersion(const char *str)
{
    HTTP::Version ver;
    ver.major = atoi(str + 5);
    ver.minor = atoi(strchr(str + 5, '.') + 1);
    return ver;
}

static
HTTP::Method
parseMethod(const char *str, const char *end)
{
	*(char*)end = '\0';
    for(size_t i = 0; i < 8; ++i) {
		if (stricmp(str, HTTP::methods[i]) == 0) {
			return (HTTP::Method)i;
		}
    }
    throw std::invalid_argument("Unrecognized method");
}

static
std::string
unfold(char *p, char *pe)
{
    char *start = p;
    char *pw = p;

    while (p < pe) {
        // Skip leading whitespace
        if (pw == start) {
            if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') {
                ++p; ++pw; ++start;
                continue;
            }
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    // Remove trailing whitespace
    do {
        --pw;
    } while ((*pw == ' ' || *pw == '\t' || *pw == '\r' || *pw == '\n') && pw >= start);
    ++pw;
    return std::string(start, pw - start);
}

static
std::string
unquote(char *p, char *pe)
{
    if (pe == p || *p != '"')
        return std::string(p, pe - p);
    char *start = p;
    char *pw = p;

    assert(*p == '"');
    assert(*(pe - 1) == '"');
    ++p; ++pw; ++start;
    --pe;
    bool escaping = false;
    while (p < pe) {
        if (escaping) {
            escaping = false;
        } else if (*p == '\\') {
            escaping = true;
            ++p;
            continue;
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    return std::string(start, pw - start);
}

%%{
    machine http_parser;

    action mark { mark = fpc; }
    action done { fbreak; }

    # basic character types
    OCTET = any;
    CHAR = ascii;
    UPALPHA = "A".."Z";
    LOALPHA = "a".."z";
    ALPHA = alpha;
    DIGIT = digit;
    CTL = cntrl | 127;
    CR = "\r";
    LF = "\n";
    SP = " ";
    HT = "\t";

    # almost-basic character types
    # note that we allow a single LF for a CR LF
    CRLF = CR LF | LF;
    LWS = CRLF? ( SP | HT )+;
    TEXT = LWS | (OCTET -- CTL);
    HEX = xdigit;

    # some basic tokens
    separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT;
    token = (CHAR -- (separators | CTL))+;
    quoted_pair = "\\" CHAR;
    ctext = TEXT -- ("(" | ")");
    base_comment = "(" ( ctext | quoted_pair )* ")";
    comment = "(" ( ctext | quoted_pair | base_comment )* ")";
    qdtext = TEXT -- "\"";
    quoted_string = "\"" ( qdtext | quoted_pair )* "\"";

    action parse_HTTP_Version {
        *m_ver = parseVersion(mark);
        mark = NULL;
    }

    HTTP_Version = ("HTTP/" DIGIT+ "." DIGIT+) >mark %parse_HTTP_Version;

    delta_seconds = DIGIT+;

    product_version = token;
    product = token ("/" product_version)?;

    qvalue = ("0" ("." DIGIT{0,3})) | ("1" ("." "0"{0,3}));

    subtag = ALPHA{1,8};
    primary_tag = ALPHA{1,8};
    language_tag = primary_tag ("-" subtag);

    weak = "W/";
    opaque_tag = quoted_string;
    entity_tag = (weak)? opaque_tag;

    bytes_unit = "bytes";
    other_range_unit = token;
    range_unit = bytes_unit | other_range_unit;

    action save_field_name {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_field_value {
        if (m_headerHandled) {
            m_headerHandled = false;
        } else {
            std::string fieldValue = unfold((char*)mark, (char*)fpc);

            StringMap::iterator it = m_entity->extension.find(m_temp1);
            if (it == m_entity->extension.end()) {
                m_entity->extension[m_temp1] = fieldValue;
            } else {
                it->second.append(", ");
                it->second.append(fieldValue);
            }
            mark = NULL;
        }
    }

    field_chars = OCTET -- (CTL | CR LF SP HT);
    field_name = token >mark %save_field_name;
    field_value = TEXT* >mark %save_field_value;
    message_header = field_name ":" field_value;

    action save_string {
        *m_string = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_ulong {
        *m_ulong = _atoi64(mark);
        mark = NULL;
    }

    action save_element {
        m_list->insert(std::string(mark, fpc - mark));
        mark = NULL;
    }
    action save_element_eof {
		m_list->insert(std::string(mark, pe - mark));
        mark = NULL;
    }
    element = token >mark %save_element %/save_element_eof;
    list = LWS* element ( LWS* ',' LWS* element)* LWS*;

    action save_parameterized_list_element {
        ValueWithParameters vp;
        vp.value = std::string(mark, fpc - mark);
        m_parameterizedList->push_back(vp);
        m_parameters = &m_parameterizedList->back().parameters;
        mark = NULL;
    }

    action save_parameter_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action save_parameter_value {
        (*m_parameters)[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    attribute = token >mark %save_parameter_attribute;
    value = token | quoted_string >mark %save_parameter_value;
    parameter = attribute '=' value;
    parameterizedListElement = token >mark %save_parameterized_list_element (';' parameter)*;
    parameterizedList = LWS* parameterizedListElement ( LWS* ',' LWS* parameterizedListElement)* LWS*;

    action set_connection {
        m_headerHandled = true;
        m_list = &m_general->connection;
    }

    action set_transfer_encoding {
        m_headerHandled = true;
        m_parameterizedList = &m_general->transferEncoding;
    }

    Connection = 'Connection:'i @set_connection list;
    Transfer_Encoding = 'Transfer-Encoding:'i @set_transfer_encoding parameterizedList;

    general_header = Connection | Transfer_Encoding;

    action set_content_length {
        m_headerHandled = true;
        m_ulong = &m_entity->contentLength;
    }

    action set_content_type
    {
		m_headerHandled = true;
		m_parameters = &m_entity->contentType.parameters;
    }
    action save_type
    {
		m_entity->contentType.type = std::string(mark, fpc - mark);
		mark = NULL;
    }
    action save_subtype
    {
		m_entity->contentType.subtype = std::string(mark, fpc - mark);
		mark = NULL;
    }

    type = token >mark %save_type;
    subtype = token >mark %save_subtype;
    media_type = type'/' subtype (';' parameter)*;

    Content_Length = 'Content-Length:'i @set_content_length LWS* DIGIT+ >mark %save_ulong LWS*;
    Content_Type = 'Content-Type:'i @set_content_type LWS* media_type LWS*;

    extension_header = message_header;

    entity_header = Content_Length | Content_Type | extension_header;

}%%

%%{
    machine http_request_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Method {
        m_request->requestLine.method = parseMethod(mark, fpc);
        mark = NULL;
    }

    action set_host {
        m_headerHandled = true;
        m_string = &m_request->request.host;
    }

    Host = 'Host:'i @set_host LWS* host (':' port)? >mark %save_string LWS*;

    request_header = Host;

    Method = token >mark %parse_Method;
    Request_URI = ( "*" | absolute_URI | hier_part | authority);
    Request_Line = Method SP Request_URI SP HTTP_Version CRLF;
    Request = Request_Line ((general_header | request_header | entity_header) CRLF)* CRLF @done;

    main := Request;
    write data;
}%%

HTTP::RequestParser::RequestParser(Request& request)
: m_request(&request),
  m_ver(&request.requestLine.ver),
  m_uri(&request.requestLine.uri),
  m_path(&request.requestLine.uri.path),
  m_general(&request.general),
  m_entity(&request.entity)
{}

void
HTTP::RequestParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::RequestParser::complete() const
{
    return cs >= http_request_parser_first_final;
}

bool
HTTP::RequestParser::error() const
{
    return cs == http_request_parser_error;
}

void
HTTP::RequestParser::exec()
{
    %% write exec;
}

%%{
    machine http_response_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Status_Code {
        m_response->status.status = (HTTP::Status)atoi(mark);
        mark = NULL;
    }

    action parse_Reason_Phrase {
        m_response->status.reason = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action set_location {
        m_headerHandled = true;
    }

    Location = 'Location:'i @set_location LWS* absolute_URI LWS*;

    response_header = Location;

    Status_Code = DIGIT{3} > mark %parse_Status_Code;
    Reason_Phrase = (TEXT -- (CR | LF))* >mark %parse_Reason_Phrase;
    Status_Line = HTTP_Version SP Status_Code SP Reason_Phrase CRLF;
    Response = Status_Line ((general_header | response_header | entity_header) CRLF)* CRLF @done;

    main := Response;

    write data;
}%%

HTTP::ResponseParser::ResponseParser(Response& response)
: m_response(&response),
  m_ver(&response.status.ver),
  m_uri(&response.response.location),
  m_path(&response.response.location.path),
  m_general(&response.general),
  m_entity(&response.entity)
{}

void
HTTP::ResponseParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::ResponseParser::complete() const
{
    return cs >= http_response_parser_first_final;
}

bool
HTTP::ResponseParser::error() const
{
    return cs == http_response_parser_error;
}

void
HTTP::ResponseParser::exec()
{
    %% write exec;
}

%%{
    machine http_trailer_parser;
    include http_parser;

    trailer = (entity_header CRLF)*;

    main := trailer CRLF @done;

    write data;
}%%

HTTP::TrailerParser::TrailerParser(EntityHeaders& entity)
: m_entity(&entity)
{}

void
HTTP::TrailerParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::TrailerParser::complete() const
{
    return cs >= http_trailer_parser_first_final;
}

bool
HTTP::TrailerParser::error() const
{
    return cs == http_trailer_parser_error;
}

void
HTTP::TrailerParser::exec()
{
    %% write exec;
}

// Copyright (c) 2009 - Decho Corp.
/* To compile to .cpp:
   ragel parser.rl -G2 -o parser.cpp
*/

#include "parser.h"

// From uri.rl
std::string unescape(const std::string& str);

static
HTTP::Version
parseVersion(const char *str)
{
    HTTP::Version ver;
    ver.major = atoi(str + 5);
    ver.minor = atoi(strchr(str + 5, '.') + 1);
    return ver;
}

static
HTTP::Method
parseMethod(const char *str, const char *end)
{
	*(char*)end = '\0';
    for(size_t i = 0; i < 8; ++i) {
		if (stricmp(str, HTTP::methods[i]) == 0) {
			return (HTTP::Method)i;
		}
    }
    throw std::invalid_argument("Unrecognized method");
}

static
std::string
unfold(char *p, char *pe)
{
    char *start = p;
    char *pw = p;

    while (p < pe) {
        // Skip leading whitespace
        if (pw == start) {
            if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') {
                ++p; ++pw; ++start;
                continue;
            }
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    // Remove trailing whitespace
    do {
        --pw;
    } while ((*pw == ' ' || *pw == '\t' || *pw == '\r' || *pw == '\n') && pw >= start);
    ++pw;
    return std::string(start, pw - start);
}

static
std::string
unquote(char *p, char *pe)
{
    if (pe == p || *p != '"')
        return std::string(p, pe - p);
    char *start = p;
    char *pw = p;

    assert(*p == '"');
    assert(*(pe - 1) == '"');
    ++p; ++pw; ++start;
    --pe;
    bool escaping = false;
    while (p < pe) {
        if (escaping) {
            escaping = false;
        } else if (*p == '\\') {
            escaping = true;
            ++p;
            continue;
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    return std::string(start, pw - start);
}

%%{
    machine http_parser;

    action mark { mark = fpc; }
    action done { fbreak; }

    # basic character types
    OCTET = any;
    CHAR = ascii;
    UPALPHA = "A".."Z";
    LOALPHA = "a".."z";
    ALPHA = alpha;
    DIGIT = digit;
    CTL = cntrl | 127;
    CR = "\r";
    LF = "\n";
    SP = " ";
    HT = "\t";

    # almost-basic character types
    # note that we allow a single LF for a CR LF
    CRLF = CR LF | LF;
    LWS = CRLF? ( SP | HT )+;
    TEXT = LWS | (OCTET -- CTL);
    HEX = xdigit;

    # some basic tokens
    separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT;
    token = (CHAR -- (separators | CTL))+;
    quoted_pair = "\\" CHAR;
    ctext = TEXT -- ("(" | ")");
    base_comment = "(" ( ctext | quoted_pair )* ")";
    comment = "(" ( ctext | quoted_pair | base_comment )* ")";
    qdtext = TEXT -- "\"";
    quoted_string = "\"" ( qdtext | quoted_pair )* "\"";

    action parse_HTTP_Version {
        *m_ver = parseVersion(mark);
        mark = NULL;
    }

    HTTP_Version = ("HTTP/" DIGIT+ "." DIGIT+) >mark %parse_HTTP_Version;

    delta_seconds = DIGIT+;

    product_version = token;
    product = token ("/" product_version)?;

    qvalue = ("0" ("." DIGIT{0,3})) | ("1" ("." "0"{0,3}));

    subtag = ALPHA{1,8};
    primary_tag = ALPHA{1,8};
    language_tag = primary_tag ("-" subtag);

    weak = "W/";
    opaque_tag = quoted_string;
    entity_tag = (weak)? opaque_tag;

    bytes_unit = "bytes";
    other_range_unit = token;
    range_unit = bytes_unit | other_range_unit;

    action save_field_name {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_field_value {
        if (m_headerHandled) {
            m_headerHandled = false;
        } else {
            std::string fieldValue = unfold((char*)mark, (char*)fpc);

            StringMap::iterator it = m_entity->extension.find(m_temp1);
            if (it == m_entity->extension.end()) {
                m_entity->extension[m_temp1] = fieldValue;
            } else {
                it->second.append(", ");
                it->second.append(fieldValue);
            }
            mark = NULL;
        }
    }

    field_chars = OCTET -- (CTL | CR LF SP HT);
    field_name = token >mark %save_field_name;
    field_value = TEXT* >mark %save_field_value;
    message_header = field_name ":" field_value;

    action save_string {
        *m_string = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_ulong {
        *m_ulong = _atoi64(mark);
        mark = NULL;
    }

    action save_element {
        m_list->insert(std::string(mark, fpc - mark));
        mark = NULL;
    }
    action save_element_eof {
		m_list->insert(std::string(mark, pe - mark));
        mark = NULL;
    }
    element = token >mark %save_element %/save_element_eof;
    list = LWS* element ( LWS* ',' LWS* element)* LWS*;

    action save_parameterized_list_element {
        ValueWithParameters vp;
        vp.value = std::string(mark, fpc - mark);
        m_parameterizedList->push_back(vp);
        m_parameters = &m_parameterizedList->back().parameters;
        mark = NULL;
    }

    action save_parameter_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action save_parameter_value {
        (*m_parameters)[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    attribute = token >mark %save_parameter_attribute;
    value = token | quoted_string >mark %save_parameter_value;
    parameter = attribute '=' value;
    parameterizedListElement = token >mark %save_parameterized_list_element (';' parameter)*;
    parameterizedList = LWS* parameterizedListElement ( LWS* ',' LWS* parameterizedListElement)* LWS*;

    action set_connection {
        m_headerHandled = true;
        m_list = &m_general->connection;
    }

    action set_transfer_encoding {
        m_headerHandled = true;
        m_parameterizedList = &m_general->transferEncoding;
    }

    Connection = 'Connection:' @set_connection list;
    Transfer_Encoding = 'Transfer-Encoding:' @set_transfer_encoding parameterizedList;

    general_header = Connection | Transfer_Encoding;

    action set_content_length {
        m_headerHandled = true;
        m_ulong = &m_entity->contentLength;
    }

    Content_Length = 'Content-Length:' @set_content_length LWS* DIGIT+ >mark %save_ulong LWS*;

    action set_content_type
    {
		m_headerHandled = true;
		m_parameters = &m_entity->contentType.parameters;
    }
    action save_type
    {
		m_entity->contentType.type = std::string(mark, fpc - mark);
		mark = NULL;
    }
    action save_subtype
    {
		m_entity->contentType.subtype = std::string(mark, fpc - mark);
		mark = NULL;
    }

    type = token >mark %save_type;
    subtype = token >mark %save_subtype;
    media_type = type'/' subtype (';' parameter)*;
    Content_Type = 'Content-Type:' @set_content_type LWS* media_type LWS*;

    extension_header = message_header;

    entity_header = Content_Length | Content_Type | extension_header;

}%%

%%{
    machine http_request_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Method {
        m_request->requestLine.method = parseMethod(mark, fpc);
        mark = NULL;
    }

    action set_host {
        m_headerHandled = true;
        m_string = &m_request->request.host;
    }

    Host = 'Host:' @set_host LWS* host (':' port)? >mark %save_string LWS*;

    request_header = Host;

    Method = token >mark %parse_Method;
    Request_URI = ( "*" | absolute_URI | hier_part | authority);
    Request_Line = Method SP Request_URI SP HTTP_Version CRLF;
    Request = Request_Line ((general_header | request_header | entity_header) CRLF)* CRLF @done;

    main := Request;
    write data;
}%%

HTTP::RequestParser::RequestParser(Request& request)
: m_request(&request),
  m_ver(&request.requestLine.ver),
  m_uri(&request.requestLine.uri),
  m_path(&request.requestLine.uri.path),
  m_general(&request.general),
  m_entity(&request.entity)
{}

void
HTTP::RequestParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::RequestParser::complete() const
{
    return cs >= http_request_parser_first_final;
}

bool
HTTP::RequestParser::error() const
{
    return cs == http_request_parser_error;
}

void
HTTP::RequestParser::exec()
{
    %% write exec;
}

%%{
    machine http_response_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Status_Code {
        m_response->status.status = (HTTP::Status)atoi(mark);
        mark = NULL;
    }

    action parse_Reason_Phrase {
        m_response->status.reason = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action set_location {
        m_headerHandled = true;
    }

    Location = 'Location:' @set_location LWS* absolute_URI LWS*;

    response_header = Location;

    Status_Code = DIGIT{3} > mark %parse_Status_Code;
    Reason_Phrase = (TEXT -- (CR | LF))* >mark %parse_Reason_Phrase;
    Status_Line = HTTP_Version SP Status_Code SP Reason_Phrase CRLF;
    Response = Status_Line ((general_header | response_header | entity_header) CRLF)* CRLF @done;

    main := Response;

    write data;
}%%

HTTP::ResponseParser::ResponseParser(Response& response)
: m_response(&response),
  m_ver(&response.status.ver),
  m_uri(&response.response.location),
  m_path(&response.response.location.path),
  m_general(&response.general),
  m_entity(&response.entity)
{}

void
HTTP::ResponseParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::ResponseParser::complete() const
{
    return cs >= http_response_parser_first_final;
}

bool
HTTP::ResponseParser::error() const
{
    return cs == http_response_parser_error;
}

void
HTTP::ResponseParser::exec()
{
    %% write exec;
}

%%{
    machine http_trailer_parser;
    include http_parser;

    trailer = (entity_header CRLF)*;

    main := trailer CRLF @done;

    write data;
}%%

HTTP::TrailerParser::TrailerParser(EntityHeaders& entity)
: m_entity(&entity)
{}

void
HTTP::TrailerParser::init()
{
    RagelParser::init();
    %% write init;
}

bool
HTTP::TrailerParser::complete() const
{
    return cs >= http_trailer_parser_first_final;
}

bool
HTTP::TrailerParser::error() const
{
    return cs == http_trailer_parser_error;
}

void
HTTP::TrailerParser::exec()
{
    %% write exec;
}


this really allows Daniel's generic typenames

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('<' . [^&]* . '>') => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    ('>,') => { fout << ">"; --p; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    '_m_tsbus_cotm_' => { fout << ">"; fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


Store macros in a map, initial substitution detection.

// Experimental parser for macros
//
// TODO:
//
// - Store macros and keys nicely in a map
// - Expand queries, marked with percent signs.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <sstream>
#include <cstring>

%%{
	machine macros;
	write data;
}%%

typedef std::map<std::string, std::string> Macros;

Macros parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;

	char const *substStart = 0;

	Macros macros;

	std::string lastKey;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		lastKey = buf.str();
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		std::string query(strStart, p - 3);
		macros[lastKey] = query;
	}

	action substBegin {
		substStart = p;
	}

	action substEnd {
		std::string subst(substStart, p);
		std::cerr << "Found substitution: " << subst << std::endl;
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	substitution = ('%' % substBegin) (any - '%')+ ('%' > substEnd);
	query = (substitution | (any - '%')+)* -- separator;
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%

	return macros;
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();
	//std::string test = "\"\"\"test\"\"\"";

	//parseMacros(test.c_str());

	Macros macros = parseMacros(data.c_str());
	for (Macros::const_iterator iter = macros.begin(); iter != macros.end(); ++iter)
		std::cout << iter->first << ": " << iter->second << std::endl;
}

// Experimental parser for macros
//
// TODO:
//
// - Store macros and keys nicely in a map
// - Expand queries, marked with percent signs.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <sstream>
#include <cstring>

%%{
	machine macros;
	write data;
}%%

void parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;
	char const *strEnd = eof;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		std::cout << "key: " << buf.str() << std::endl;
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		std::string query(strStart, p - 3);
		std::cout << "value: " << query << std::endl;
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	query = (any* -- separator);
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();

	parseMacros(data.c_str());
}


Actually parse the Session header, fixes other headers parsing.

/* -*- c -*- */

%% machine rtsp_request;

static int ragel_parse_request(RTSP_Request *req, char *msg) {
    int cs;
    char *p = msg, *pe = p + strlen(msg), *eof, *s;

    /* Variables used for adding headers to the hash table */
    char *hdr, *hdr_val;
    size_t hdr_size, hdr_val_size;

    /* We want to express clearly which versions we support, so that we
     * can return right away if an unsupported one is found.
     */

    %%{
        SP = ' ';
        CRLF = "\r\n";

        action set_s {
            s = p;
        }

        action end_method {
            req->method = g_strndup(s, p-s);
        }

        Supported_Method =
            "DESCRIBE" @ { req->method_id = RTSP_ID_DESCRIBE; } |
            "OPTIONS" @ { req->method_id = RTSP_ID_OPTIONS; } |
            "PAUSE" @ { req->method_id = RTSP_ID_PAUSE; } |
            "PLAY" @ { req->method_id = RTSP_ID_PLAY; } |
            "SETUP" @ { req->method_id = RTSP_ID_SETUP; } |
            "SET_PARAMETER" @ { req->method_id = RTSP_ID_SET_PARAMETER; } |
            "TEARDOWN" @ { req->method_id = RTSP_ID_TEARDOWN; };

        action method_not_implemented {
            return RTSP_NotImplemented;
        }

        Other_Method = (alpha+ - Supported_Method) %method_not_implemented;
        Method = (Supported_Method | Other_Method )
            > set_s % end_method;

        action version_not_supported {
            return RTSP_VersionNotSupported;
        }

        Supported_RTSP_Version = "RTSP/1.0";
        RTSP_Version = (alpha . '/' . [0-9] '.' [0-9]) -
            Supported_RTSP_Version % version_not_supported;

        action end_object {
            req->object = g_strndup(s, p-s);
        }

        Request_Line = (Supported_Method | Method) . SP
            (print*) > set_s % end_object . SP
            (Supported_RTSP_Version | RTSP_Version) . CRLF;

        action hdr_start {
            hdr = p;
        }

        action hdr_end {
            hdr_size = p-hdr;
        }

        action hdr_val_start {
            hdr_val = p;
        }

        action hdr_val_end {
            hdr_val_size = p-hdr_val;
        }

        Header = (alpha|'-')+ > hdr_start % hdr_end
            . ':' . SP . print+ > hdr_val_start % hdr_val_end . CRLF;

        action cseq_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->cseq = strtol(s, NULL, 0);
                g_free(tmp);
            }
        }

        Cseq_Header = "CSeq: " . (digit+ >set_s) . CRLF %cseq_header;

        action session_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->session_id = g_ascii_strtoull(s, NULL, 0);
                g_free(tmp);
            }
        }

        Session_Header = "Session: " . (digit+ >set_s) . CRLF %session_header;

        Known_Headers = Cseq_Header | Session_Header;
        Other_Headers = Header - Known_Headers;

        action save_header {
            {
                g_hash_table_insert(req->headers,
                                    g_strndup(hdr, hdr_size),
                                    g_strndup(hdr_val, hdr_val_size));
            }
        }

        main := Request_Line . ( Known_Headers | Other_Headers @ save_header )+;

        write data;
        write init;
        write exec;
    }%%

    return RTSP_Ok;
}

/* -*- c -*- */

%% machine rtsp_request;

static int ragel_parse_request(RTSP_Request *req, char *msg) {
    int cs;
    char *p = msg, *pe = p + strlen(msg), *eof, *s;

    /* Variables used for adding headers to the hash table */
    char *hdr, *hdr_val;
    size_t hdr_size, hdr_val_size;

    /* We want to express clearly which versions we support, so that we
     * can return right away if an unsupported one is found.
     */

    %%{
        SP = ' ';
        CRLF = "\r\n";

        action set_s {
            s = p;
        }

        action end_method {
            req->method = g_strndup(s, p-s);
        }

        Supported_Method =
            "DESCRIBE" @ { req->method_id = RTSP_ID_DESCRIBE; } |
            "OPTIONS" @ { req->method_id = RTSP_ID_OPTIONS; } |
            "PAUSE" @ { req->method_id = RTSP_ID_PAUSE; } |
            "PLAY" @ { req->method_id = RTSP_ID_PLAY; } |
            "SETUP" @ { req->method_id = RTSP_ID_SETUP; } |
            "SET_PARAMETER" @ { req->method_id = RTSP_ID_SET_PARAMETER; } |
            "TEARDOWN" @ { req->method_id = RTSP_ID_TEARDOWN; };

        action method_not_implemented {
            return RTSP_NotImplemented;
        }

        Other_Method = (alpha+ - Supported_Method) %method_not_implemented;
        Method = (Supported_Method | Other_Method )
            > set_s % end_method;

        action version_not_supported {
            return RTSP_VersionNotSupported;
        }

        Supported_RTSP_Version = "RTSP/1.0";
        RTSP_Version = (alpha . '/' . [0-9] '.' [0-9]) -
            Supported_RTSP_Version % version_not_supported;

        action end_object {
            req->object = g_strndup(s, p-s);
        }

        Request_Line = (Supported_Method | Method) . SP
            (print*) > set_s % end_object . SP
            (Supported_RTSP_Version | RTSP_Version) . CRLF;

        action hdr_start {
            hdr = p;
        }

        action hdr_end {
            hdr_size = p-hdr;
        }

        action hdr_val_start {
            hdr_val = p;
        }

        action hdr_val_end {
            hdr_val_size = p-hdr_val;
        }

        Header = (alpha|'-')+ > hdr_start % hdr_end
            . ':' . SP . print+ > hdr_val_start % hdr_val_end . CRLF;

        action cseq_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->cseq = strtol(s, NULL, 0);
                g_free(tmp);
            }
        }

        Cseq_Header = "CSeq: " . (digit+ >set_s) . CRLF %cseq_header;

        action session_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->session_id = g_strtoull(s, NULL, 0);
                g_free(tmp);
            }
        }

        Session_Header = "Session: " . (digit+ >set_s) . CRLF %session_header;

        Known_Headers = Cseq_Header | Session_Header;
        Other_Headers = Header - Known_Headers;

        action save_header {
            {
                g_hash_table_insert(req->headers,
                                    g_strndup(hdr, hdr_size),
                                    g_strndup(hdr_val, hdr_val_size));
            }
        }

        main := Request_Line . ( Cseq_Header | Other_Headers % save_header )+;

        write data;
        write init;
        write exec;
    }%%

    return RTSP_Ok;
}


state charts

state charts

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>

#include "toml.h"

%%{
	machine toml;

	action a_boolean	{
		printf("got boolean %.*s\n", ts[0] == 't' ? 4 : 5, ts);
		fret;
	}

	action a_integer	{
		int64_t integer = strtol(ts, NULL, 10);
		printf("%d got integer %"PRId64"\n", cs, integer);
		printf("stack top is %d -> %d\n", top - 1, stack[top - 1]);
		printf("p = '%.5s'\n", p);
		fret;
	}

	action a_float		{
		double floating = strtod(ts, NULL);
		printf("got float %lf\n", floating); fret;
	}

	action a_string		{
		printf("got string %.*s\n", (int)(te-ts), ts);
		fret;
	}

	action a_date		{
		struct tm tm;
		time_t t;

		sscanf(ts, "%4d-%2d-%2dT%2d:%2d:%2dZ", &tm.tm_year, &tm.tm_mon,
						&tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec);

		tm.tm_year -= 1900;
		tm.tm_mon -= 1;
		tm.tm_zone = "UTC";
		tm.tm_gmtoff = 0;

		t = timegm(&tm);
		printf("got date %d\n", (int)t);

		printf("p = '%.5s'\n", p);

		fret;
	}

	#utf8 := print - '"' @return;
	whitespace = ([\t ]* ${ printf("got ws char %d cs = %d p = %p p5 = '%.5s' nacts = %d\n", *p, cs, p, p, _nacts); });

	boolean = ('true'|'false');
	string	= '"' (print - '"')* '"';
	integer	= [+\-]?digit+;
	float	= [+\-]?digit+ '.' digit+;
	date	= digit{4} '-' digit{2} '-' digit{2} 'T' digit{2} ':' digit{2} ':'
				digit{2} 'Z';
	log = any ${printf("log got '%.5s'\n", p);};

	value := |*
		boolean => a_boolean;
		string	=> a_string;
		float	=> a_float;
		integer	=> a_integer;
		date	=> a_date;
	*|;

	comment = '#' [^\n]* %{ printf("got comment on %d\n", curline); };

	key = (print - (whitespace|']'))+ >{key = p;} %{ printf("key is %.*s\n", (int)(1+p-key), key);};

	nl = '\n' @{ printf("got newline\n"); curline += 1; };

	line = (
		start: (
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		text: (
			'#'	@{fhold;}			->final		|
			'[' 					->map		|
			[\t ]					->text		|
			'\n' %{fhold;}			->final		|
			[^#[\t \n] @{fhold;}	->keyval	|
			''						->final
		),

		map: (
			key ']' ->text
		),

		aval: (
			'true'			-> true				|
			'false'			-> false			|
			'"' 			-> string			|
			'-'				-> negative_number	|
			'+'				-> number			|
			digit			-> number_or_date
		),

#		aval: (
#			any >{
#				printf("cs %d fcurs %d entering value p = '%.5s'\n", cs, fcurs, p);
#				fcall value;
#			}
#		   	->final
#		),

		newlist: (
			[\t ]+						->newlist	|
			[^\t \]] >{fcall value;}	->list		|
			']'							->text
		),

		val: (
			'['													->newlist	|
			'\n' @{fhold;}										->text		|
			[^[\n] >{fhold;printf("entering aval p = '%.5s'\n", p);}	->aval
		),

		list: (
			[\t ]+					->list	|
			','						->list 	|
			']'						->text  |
			[^\t ,\]] >{fhold;}		->val
	  	)

		keyval: (
			key whitespace '=' @{printf("got equals\n");} whitespace !whitespace >{printf("not whitespace p='%.5s'\n", p);}->val
		)
	);

	main := (line comment? nl)*;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, act, curline = 0;
	int stack[100], top;
	char *ts, *te, *eof = NULL, *p, *pe;
	char *key;

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR, p = '%.5s'", p);
		return 1;
	}

	return 0;
}

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>

#include "toml.h"

%%{
	machine toml;

	action ResetIndent {
		indent = 0;
	}
	action CountIndent {
		indent++;
	}
	action Indented {
		printf("indent is %d on %d\n", indent, curline);
	}

	action return { fret; }

	action a_boolean	{
		printf("got boolean %.*s\n", ts[0] == 't' ? 4 : 5, ts);
		fret;
	}

	action a_integer	{
		int64_t integer = strtol(ts, NULL, 10);
		printf("got integer %"PRId64"\n", integer);
		printf("stack top is %d -> %d\n", top - 1, stack[top - 1]);
		printf("p = '%.5s'\n", p);
		fret;
	}

	action a_float		{
		double floating = strtod(ts, NULL);
		printf("got float %lf\n", floating); fret;
	}

	action a_string		{
		printf("got string %.*s\n", (int)(te-ts), ts);
		fret;
	}

	action a_date		{
		struct tm tm;
		time_t t;

		sscanf(ts, "%4d-%2d-%2dT%2d:%2d:%2dZ", &tm.tm_year, &tm.tm_mon,
						&tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec);

		tm.tm_year -= 1900;
		tm.tm_mon -= 1;
		tm.tm_zone = "UTC";
		tm.tm_gmtoff = 0;

		t = timegm(&tm);
		printf("got date %d\n", (int)t);

		printf("p = '%.5s'\n", p);

		fret;
	}

	action a_list		{ printf("got a list cs = %d\n", cs); fret; }

	utf8 := print - '"' @return;
	whitespace = ([\t ]* ${ printf("got ws char %d cs = %d p = %p p5 = '%.5s' nacts = %d\n", *p, cs, p, p, _nacts); });

	boolean = ('true'|'false');
	string	= '"' (print - '"')* '"';
	integer	= [+\-]?digit+;
	float	= [+\-]?digit+ '.' digit+;
	date	= digit{4} '-' digit{2} '-' digit{2} 'T' digit{2} ':' digit{2} ':'
				digit{2} 'Z';

#	value = (boolean|integer|float|string|date);
#		list	=> a_list;

	action call_value1 {
		printf("1cs %d, fcurs %d, p %d\n", cs, fcurs, *p);
		printf("1stack top is %d -> %d\n", top-1, stack[top-1]);
		printf("1value is state %d\n", fentry(value));
		stack[top++] = cs;
		fgoto value;
	}
	action call_value2 {
		printf("2cs %d, fcurs %d, p %d\n", cs, fcurs, *p);
		printf("2stack top is %d -> %d\n", top-1, stack[top-1]);
		printf("2value is state %d\n", fentry(value));
		stack[top++] = cs;
		fgoto value;
	}

#list_item = (whitespace %call_value) whitespace;
	list = '[' %{printf("got open list\n");} (whitespace |
			whitespace :> !whitespace >call_value1 whitespace
			(',' >{printf("got comma\n");} whitespace @{printf("ws3\n");} :> !whitespace >call_value2 whitespace >{printf("ws4\n");})*)?
		']';

#(whitespace %call_value whitespace (',' whitespace %call_value whitespace)* )? whitespace ']';
	value := |*
		boolean => a_boolean;
		string	=> a_string;
		float	=> a_float;
		integer	=> a_integer;
		date	=> a_date;
#'[' (whitespace | whitespace @call_value1 whitespace
#				(',' %{printf("got comma\n");} whitespace @call_value2 whitespace)*) ']' {};
		list	=> a_list;
#list;
	*|;

	comment = '#' [^\n]* %{ printf("got comment on %d\n", curline); };

	key = (print - (whitespace|']'))+ >{ ts = p; } %{ snprintf(key, 1+ p - ts, "%s", ts); };

	indentation = whitespace >ResetIndent $CountIndent @Indented;

	nl = '\n' @{ printf("got newline\n"); curline += 1; };

	line = (
		whitespace comment %{
			printf("comment on %d\n", curline);
		} |

		whitespace %{
			printf("empty line on %d\n", curline);
		} |

		indentation '[' whitespace key whitespace ']' whitespace comment? %{
			printf("map [%s] on %d\n", key, curline);
		} |

		indentation whitespace key whitespace ('=' %{ printf("got equals\n"); } ) whitespace >{printf("entering ws1\n");} :> !whitespace >{fcall value;} whitespace >{printf("entering ws2 p = %d\n", *p);} comment? >{printf("entering comment p = %d\n", *p);} %{
			printf("key %s on %d\n", key, curline);
		}
	);

	main := (line nl)*;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, act, curline = 0;
	int stack[100], top;
	char *ts, *te, *eof = NULL, *p, *pe;
	char key[100];

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (cs == toml_error) {
		fprintf(stderr, "PARSE_ERROR");
		return 1;
	}

	return 0;
}


Ragel: refactoring and debug switch

Ragel: refactoring and debug switch

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call(:string,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call(:integer,data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call(:float,data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call(:identifier,data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call(:bool,data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  p s
  VcfHeader::RagelKeyValues.run_lexer(s, debug: true)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call(:string,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call(:integer,data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call(:float,data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call(:identifier,data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit.call("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit.call("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}"
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| VcfHeader::RagelKeyValues.run_lexer(s) }

end