Updated state machine that fixes the duplicate registrations problem.

/**
 *
 * Copyright (c) 2010, Zed A. Shaw and Mongrel2 Project Contributors.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 *     * Neither the name of the Mongrel2 Project, Zed A. Shaw, nor the names
 *       of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

%%{
    machine State;

    import "events.h";

Proxy := (
        start: (
           CONNECT @proxy_deliver -> Sending |
           FAILED @proxy_failed -> Closing
        ),

        Proxying: (
            HTTP_REQ @proxy_deliver -> Sending |
            PROXY @proxy_exit_routing |
            HANDLER @proxy_exit_routing |
            DIRECTORY @proxy_exit_routing |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Sending: (
            REQ_SENT @proxy_reply_parse -> Receiving |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Receiving: (
            REQ_RECV @proxy_req_parse -> Proxying |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Closing: (
            CLOSE @proxy_exit_idle
        )

     )  <err(error);

Connection = (
        start: ( OPEN @open -> Accepting ),

        Accepting: ( ACCEPT @parse -> Identifying ),

        Identifying: (
            REQ_RECV @register_request -> Registered |
            CLOSE @close -> final
        ),

        Registered: (
            HTTP_REQ @route_request -> HTTPRouting |
            MSG_REQ @route_request -> MSGRouting |
            SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        Idle: (
            REQ_RECV @identify_request HTTP_REQ @route_request -> HTTPRouting |
            REQ_RECV @identify_request MSG_REQ @route_request -> MSGRouting |
            REQ_RECV @identify_request SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        MSGRouting: ( HANDLER @msg_to_handler -> Queueing ),

        HTTPRouting: (
            HANDLER @http_to_handler -> Queueing |
            PROXY @http_to_proxy  |
            DIRECTORY @http_to_directory -> Responding |
            CLOSE @close -> final
        ),

        Queueing: ( REQ_SENT @parse -> Idle ),

        Responding: (
            RESP_SENT @parse -> Idle |
            CLOSE @close -> final
        )

    ) %eof(finish) <err(error);

main := (Connection)*;

}%%

/**
 *
 * Copyright (c) 2010, Zed A. Shaw and Mongrel2 Project Contributors.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 *     * Neither the name of the Mongrel2 Project, Zed A. Shaw, nor the names
 *       of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

%%{
    machine State;

    import "events.h";

Proxy := (
        start: (
           CONNECT @proxy_deliver -> Sending |
           FAILED @proxy_failed -> Closing
        ),

        Proxying: (
            HTTP_REQ @proxy_deliver -> Sending |
            PROXY @proxy_exit_routing |
            HANDLER @proxy_exit_routing |
            DIRECTORY @proxy_exit_routing |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Sending: (
            REQ_SENT @proxy_reply_parse -> Receiving |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Receiving: (
            REQ_RECV @proxy_req_parse -> Proxying |
            REMOTE_CLOSE @proxy_close -> Closing
        ),

        Closing: (
            CLOSE @proxy_exit_idle
        )

     )  <err(error);

Connection = (
        start: ( OPEN @open -> Accepting ),

        Accepting: ( ACCEPT @parse -> Idle ),

        Idle: (
            REQ_RECV @identify_request HTTP_REQ @route_request -> HTTPRouting |
            REQ_RECV @identify_request MSG_REQ @route_request -> MSGRouting |
            REQ_RECV @identify_request SOCKET_REQ @send_socket_response -> Responding |
            CLOSE @close -> final
        ),

        MSGRouting: ( HANDLER @msg_to_handler -> Queueing ),

        HTTPRouting: (
            HANDLER @http_to_handler -> Queueing |
            PROXY @http_to_proxy  |
            DIRECTORY @http_to_directory -> Responding |
            CLOSE @close -> final
        ),

        Queueing: ( REQ_SENT @parse -> Idle ),

        Responding: (
            RESP_SENT @parse -> Idle |
            CLOSE @close -> final
        )

    ) %eof(finish) <err(error);

main := (Connection)*;

}%%


adjusted conderr.rl expected output following bug fix in parent commit

/*
 * @LANG: c++
 */

#include <iostream>
#include <string.h>
using std::cout;
using std::endl;

%%{
	machine foo;

	action c {c}
	main :=
		( ( 'a' when c )+ ) $err{ cout << "  bail" << endl; }
		'\n';
}%%

%% write data noerror;

void test( int c, const char *str )
{
	int cs = foo_start;
	const char *p = str;
	const char *pe = str + strlen( str );
	const char *eof = pe;

	cout << "run:" << endl;
	%% write exec;
	if ( cs >= foo_first_final )
		cout << "  success" << endl;
	else
		cout << "  failure" << endl;
	cout << endl;
}

int main()
{
	test( 1, "aa\n" );
	test( 1, "ab\n" );
	test( 0, "aa\n" );
	test( 0, "ab\n" );
	return 0;
}

#ifdef _____OUTPUT_____
run:
  success

run:
  bail
  failure

run:
  bail
  failure

run:
  bail
  failure

#endif

/*
 * @LANG: c++
 */

#include <iostream>
#include <string.h>
using std::cout;
using std::endl;

%%{
	machine foo;

	action c {c}
	main :=
		( ( 'a' when c )+ ) $err{ cout << "  bail" << endl; }
		'\n';
}%%

%% write data noerror;

void test( int c, const char *str )
{
	int cs = foo_start;
	const char *p = str;
	const char *pe = str + strlen( str );
	const char *eof = pe;

	cout << "run:" << endl;
	%% write exec;
	if ( cs >= foo_first_final )
		cout << "  success" << endl;
	else
		cout << "  failure" << endl;
	cout << endl;
}

int main()
{
	test( 1, "aa\n" );
	test( 1, "ab\n" );
	test( 0, "aa\n" );
	test( 0, "ab\n" );
	return 0;
}

#ifdef _____OUTPUT_____
run:
  bail
  bail
  success

run:
  bail
  bail
  failure

run:
  bail
  failure

run:
  bail
  failure

#endif


Fix naive implementation of X for pack.

Fix naive implementation of X for pack.

// vim: filetype=ragel

%%{

  machine pack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    rest = true;
  }

  action platform {
    platform = true;
  }

  action check_size {
    stop = rest ? size() : index + count;
    if(stop > size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action string_check_size {
    if(index >= size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action C {
    PACK_INT_ELEMENTS(MASK_BYTE);
  }

  action S {
    PACK_INT_ELEMENTS(MASK_16BITS);
  }

  action I {
    PACK_INT_ELEMENTS(MASK_32BITS);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      PACK_INT_ELEMENTS(MASK_32BITS);
#else
      PACK_LONG_ELEMENTS(MASK_64BITS);
#endif
    } else {
      PACK_INT_ELEMENTS(MASK_32BITS);
    }
  }

  action n {
    PACK_INT_ELEMENTS(BE_MASK_16BITS);
  }

  action N {
    PACK_INT_ELEMENTS(BE_MASK_32BITS);
  }

  action v {
    PACK_INT_ELEMENTS(LE_MASK_16BITS);
  }

  action V {
    PACK_INT_ELEMENTS(LE_MASK_32BITS);
  }

  action Q {
    PACK_LONG_ELEMENTS(MASK_64BITS);
  }

  action X {
#define INVALID_MOVE_ERROR_SIZE 48

    if(rest) count = 0;

    if(count > str.size()) {
      char invalid_move_msg[INVALID_MOVE_ERROR_SIZE];
      snprintf(invalid_move_msg, INVALID_MOVE_ERROR_SIZE,
               "X%d exceeds length of string", (int)count);
      Exception::argument_error(state, invalid_move_msg);
    }

    str.resize(str.size() - count);
  }

  action x {
    if(rest) count = 0;

    str.append(count, '\0');
  }

  action at {
    if(rest) count = 1;

    if(count > str.size()) {
      str.append(count - str.size(), '\0');
    } else {
      str = str.substr(0, count);
    }
  }

  action A {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(count > 0) str.append(count, ' ');
  }

  action a {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(count > 0) str.append(count, '\0');
  }

  action Z {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(rest) {
      if(count == 0) str.append(1, '\0');
    } else {
      if(count > 0) str.append(count, '\0');
    }
  }

  action fail {
    return force_as<String>(Primitives::failure());
  }

  action non_native_error {
#define NON_NATIVE_ERROR_SIZE 36

    char non_native_msg[NON_NATIVE_ERROR_SIZE];
    snprintf(non_native_msg, NON_NATIVE_ERROR_SIZE,
             "'%c' allowed only after types sSiIlL", *p);
    Exception::argument_error(state, non_native_msg);
  }

  action done {
    String* result = String::create(state, str.c_str(), str.size());
    if(tainted) {
      result->taint(state);
      tainted = false;
    }
    return result;
  }
}%%

// vim: filetype=ragel

%%{

  machine pack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    rest = true;
  }

  action platform {
    platform = true;
  }

  action check_size {
    stop = rest ? size() : index + count;
    if(stop > size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action string_check_size {
    if(index >= size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action C {
    PACK_INT_ELEMENTS(MASK_BYTE);
  }

  action S {
    PACK_INT_ELEMENTS(MASK_16BITS);
  }

  action I {
    PACK_INT_ELEMENTS(MASK_32BITS);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      PACK_INT_ELEMENTS(MASK_32BITS);
#else
      PACK_LONG_ELEMENTS(MASK_64BITS);
#endif
    } else {
      PACK_INT_ELEMENTS(MASK_32BITS);
    }
  }

  action n {
    PACK_INT_ELEMENTS(BE_MASK_16BITS);
  }

  action N {
    PACK_INT_ELEMENTS(BE_MASK_32BITS);
  }

  action v {
    PACK_INT_ELEMENTS(LE_MASK_16BITS);
  }

  action V {
    PACK_INT_ELEMENTS(LE_MASK_32BITS);
  }

  action Q {
    PACK_LONG_ELEMENTS(MASK_64BITS);
  }

  action X {
#define INVALID_MOVE_ERROR_SIZE 48

    if(rest) count = 0;

    if(count > str.size()) {
      char invalid_move_msg[INVALID_MOVE_ERROR_SIZE];
      snprintf(invalid_move_msg, INVALID_MOVE_ERROR_SIZE,
               "X%d exceeds length of string", (int)count);
      Exception::argument_error(state, invalid_move_msg);
    }

    str = str.substr(0, str.size() - count);
  }

  action x {
    if(rest) count = 0;

    str.append(count, '\0');
  }

  action at {
    if(rest) count = 1;

    if(count > str.size()) {
      str.append(count - str.size(), '\0');
    } else {
      str = str.substr(0, count);
    }
  }

  action A {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(count > 0) str.append(count, ' ');
  }

  action a {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(count > 0) str.append(count, '\0');
  }

  action Z {
    PACK_STRING_ELEMENT(pack::string_or_nil);
    if(rest) {
      if(count == 0) str.append(1, '\0');
    } else {
      if(count > 0) str.append(count, '\0');
    }
  }

  action fail {
    return force_as<String>(Primitives::failure());
  }

  action non_native_error {
#define NON_NATIVE_ERROR_SIZE 36

    char non_native_msg[NON_NATIVE_ERROR_SIZE];
    snprintf(non_native_msg, NON_NATIVE_ERROR_SIZE,
             "'%c' allowed only after types sSiIlL", *p);
    Exception::argument_error(state, non_native_msg);
  }

  action done {
    String* result = String::create(state, str.c_str(), str.size());
    if(tainted) {
      result->taint(state);
      tainted = false;
    }
    return result;
  }
}%%


any+ wrong handling??

any+ wrong handling??

%%{

  machine lexer;

  access @;

  T_KEYWORD_DEF = '->';
  T_KEYWORD_END = '<-';
  T_DQUOTE      = '"';
  T_INT         = ('+'|'-')?[0-9]+;
  T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
  T_LITERAL     = [a-zA-Z_][a-zA-Z_0-9]*;
  T_ADD         = '+';
  T_SUB         = '-';
  T_MUL         = '*';
  T_DIV         = '/';
  T_MOD         = '%';
  T_EXP         = '^';
  T_LBR         = '(';
  T_RBR         = ')';
  T_EQ          = '=';
  T_COL         = ',';

  main := |*

    T_KEYWORD_DEF => {
      emit(:T_KEYWORD_DEF)
    };

    T_KEYWORD_END => {
      emit(:T_KEYWORD_END)
    };

    T_DQUOTE => {
      emit(:T_DQUOTE)
    };

    T_INT => {
      emit(:T_INT)
    };

    T_FLOAT => {
      emit(:T_FLOAT)
    };

    T_LITERAL => {
      emit(:T_LITERAL)
    };

    T_ADD => {
      emit(:T_ADD)
    };

    T_SUB => {
      emit(:T_SUB)
    };

    T_MUL => {
      emit(:T_MUL)
    };

    T_DIV => {
      emit(:T_DIV)
    };

    T_EXP => {
      emit(:T_EXP)
    };

    T_MOD => {
      emit(:T_MOD)
    };

    T_LBR => {
      emit(:T_LBR)
    };

    T_RBR => {
      emit(:T_RBR)
    };

    T_EQ => {
      emit(:T_EQ)
    };

    T_COL => {
      emit(:T_COL)
    };

    space;

    any+ => {
      emit(:UNKNOWN)
    };

  *|;

}%%

module CakeLang
  class Lexer
      CONVERSION = {
          :T_INT => :to_i,
          :T_FLOAT => :to_f
        }

      %% write data;

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      def initialize
        reset
      end

      def reset
        @data     = nil
        @ts       = nil
        @te       = nil
        @tokens   = []
       end

      def emit(symbol)
        # @ts, @te -> token start, token end
        # had to use @te-1 for token range end because 1 digit token (eg.: ',(,),+) and no whitespace between will
        # break the tokenization
        value = @data[@ts..(@te-1)].pack("U*").strip

        if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
          value = value.send(CONVERSION[symbol])
        end

        @tokens << [symbol, value]
      end

      def lex(data)
        @data = data.unpack("c*")
        lexer_start = self.class.lexer_start
        eof = data.length

        %% write init;
        %% write exec;

        tokens = @tokens

        reset

        return tokens
      end
  end
end

%%{

  machine lexer;

  access @;

  T_KEYWORD_DEF = '->';
  T_KEYWORD_END = '<-';
  T_DQUOTE      = '"';
  T_INT         = ('+'|'-')?[0-9]+;
  T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
  T_LITERAL     = [a-zA-Z_][a-zA-Z_0-9]*;
  T_ADD         = '+';
  T_SUB         = '-';
  T_MUL         = '*';
  T_DIV         = '/';
  T_MOD         = '%';
  T_EXP         = '^';
  T_LBR         = '(';
  T_RBR         = ')';
  T_EQ          = '=';
  T_COL         = ',';

  main := |*

    T_KEYWORD_DEF => {
      emit(:T_KEYWORD_DEF)
    };

    T_KEYWORD_END => {
      emit(:T_KEYWORD_END)
    };

    T_DQUOTE => {
      emit(:T_DQUOTE)
    };

    T_INT => {
      emit(:T_INT)
    };

    T_FLOAT => {
      emit(:T_FLOAT)
    };

    T_LITERAL => {
      emit(:T_LITERAL)
    };

    T_ADD => {
      emit(:T_ADD)
    };

    T_SUB => {
      emit(:T_SUB)
    };

    T_MUL => {
      emit(:T_MUL)
    };

    T_DIV => {
      emit(:T_DIV)
    };

    T_EXP => {
      emit(:T_EXP)
    };

    T_MOD => {
      emit(:T_MOD)
    };

    T_LBR => {
      emit(:T_LBR)
    };

    T_RBR => {
      emit(:T_RBR)
    };

    T_EQ => {
      emit(:T_EQ)
    };

    T_COL => {
      emit(:T_COL)
    };

    space;

  *|;

}%%

module CakeLang
  class Lexer
      CONVERSION = {
          :T_INT => :to_i,
          :T_FLOAT => :to_f
        }

      %% write data;

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      def initialize
        reset
      end

      def reset
        @data     = nil
        @ts       = nil
        @te       = nil
        @tokens   = []
       end

      def emit(symbol)
        # @ts, @te -> token start, token end
        # had to use @te-1 for token range end because 1 digit token (eg.: ',(,),+) and no whitespace between will
        # break the tokenization
        value = @data[@ts..(@te-1)].pack("U*").strip

        if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
          value = value.send(CONVERSION[symbol])
        end

        @tokens << [symbol, value]
      end

      def lex(data)
        @data = data.unpack("c*")
        lexer_start = self.class.lexer_start
        eof = data.length

        %% write init;
        %% write exec;

        tokens = @tokens

        reset

        return tokens
      end
  end
end


Start fleshing out the CParser by parsing commands

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define MARK_STR_NEW() rb_str_new(mark, MARK_LEN)

#define true 1
#define false 0

typedef struct {
  VALUE error;

  size_t max_message_size;

  const char *chunk;
  const char *p;
  int cs;
  const char *mark;
  const char *mark_key;
  VALUE mark_message;
  size_t mark_message_size;
  size_t mark_message_content_length;
} parser_state_t;

VALUE mStomp = Qnil;
VALUE cMessage = Qnil;
ID g_new;
ID g_write_command;
ID g_build_parse_error;
ID g_max_message_size;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = rb_funcall(cMessage, g_new, 2, Qnil, Qnil);
    mark_message_size = 0;
  }

  action write_command {
    rb_funcall(mark_message, g_write_command, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action mark_key {
    /*
    mk = MARK_STR_NEW;
    mark = NULL;
    */
  }

  action write_header {
    /*
    VALUE value = MARK_STR_NEW;
    mark = NULL;

    rb_funcall(message, rb_intern("write_header"), 2, mk, value);
    mk = Qnil;
    */
  }

  action finish_headers {
    /*
    VALUE length = rb_funcall(message, rb_intern("content_length"), 0);
    if (FIXNUM_P(length)) {
      content_length = FIX2LONG(length);
    } else {
      content_length = -1;
    }
    */
  }

  action write_body {
    /*
    VALUE body = MARK_STR_NEW;
    mark = NULL;
    rb_funcall(message, rb_intern("write_body"), 1, body);
    */
  }

  action consume_null {
    false
    /* ((content_length != -1) && (MARK_LEN < content_length)) */
  }

  action consume_octet {
    true
    /* ((content_length == -1) || (MARK_LEN < content_length)) */
  }

  action check_message_size {
    mark_message_size += 1;
  }

  action finish_message {
    rb_yield(mark_message);
    mark_message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->mark_message);
  rb_gc_mark(state->error);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  VALUE max_message_size;
  rb_scan_args(argc, argv, "01", &max_message_size);

  if (max_message_size == Qnil) {
    max_message_size = rb_funcall(mStomp, g_max_message_size, 0);
  }

  state->error = Qnil;
  state->max_message_size = FIX2LONG(max_message_size);
  state->chunk = NULL;
  state->p = NULL;
  state->cs = start;
  state->mark = NULL;
  state->mark_key = NULL;
  state->mark_message = Qnil;
  state->mark_message_size = 0;
  state->mark_message_content_length = 0;

  return self;
}

static VALUE parser_parse(VALUE self, VALUE chunk) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  if (NIL_P(state->error)) {
    /*
    if state.chunk
      p = state.chunk.bytesize
      chunk = state.chunk << chunk
    else
      p = 0
    end

    pe = chunk.bytesize # special
    */

    // size_t max_message_size = state->max_message_size;
    const char *p = RSTRING_PTR(chunk);
    const char *pe = p + RSTRING_LEN(chunk);

    int cs = state->cs;
    const char *mark = state->mark;
    // mark_key = state->mark_key;
    VALUE mark_message = state->mark_message;
    size_t mark_message_size = state->mark_message_size;
    // mark_content_length = state->mark_content_length;

    %% write exec;

    /*
    if mark
      state.p = chunk.bytesize
      state.chunk = chunk
    else
      state.p = 0
      state.chunk = nil
    end
    */

    state->cs = cs;
    state->mark = mark;
    // state->mark_key = mark_key;
    state->mark_message = mark_message;
    state->mark_message_size = mark_message_size;
    // state->mark_content_length = mark_content_length;

    if (cs == error) {
      long index = p - RSTRING_PTR(chunk);
      state->error = rb_funcall(mStomp, g_build_parse_error, 2, chunk, LONG2NUM(index));
    }
  }

  if ( ! NIL_P(state->error)) {
    rb_exc_raise(state->error);
  }

  return Qnil;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));

  mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));

  g_new = rb_intern("new");
  g_write_command = rb_intern("write_command");
  g_build_parse_error = rb_intern("build_parse_error");
  g_max_message_size = rb_intern("max_message_size");

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "parse", parser_parse, 1);
}

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define READ(obj, prop) rb_funcall((obj), rb_intern((prop)), 0)
#define MARK_STR_NEW rb_str_new(mark, MARK_LEN)

#define true 1
#define false 0

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    message = rb_funcall(cMessage, rb_intern("new"), 2, Qnil, Qnil);
    message_size = 0;
  }

  action write_command {
    VALUE command = rb_str_new(mark, p - mark);
    rb_funcall(message, rb_intern("write_command"), 1, command);
    mark = NULL;
  }

  action mark_key {
    mk = MARK_STR_NEW;
    mark = NULL;
  }

  action write_header {
    VALUE value = MARK_STR_NEW;
    mark = NULL;

    rb_funcall(message, rb_intern("write_header"), 2, mk, value);
    mk = Qnil;
  }

  action finish_headers {
    VALUE length = rb_funcall(message, rb_intern("content_length"), 0);
    if (FIXNUM_P(length)) {
      content_length = FIX2LONG(length);
    } else {
      content_length = -1;
    }
  }

  action write_body {
    VALUE body = MARK_STR_NEW;
    mark = NULL;
    rb_funcall(message, rb_intern("write_body"), 1, body);
  }

  action consume_null {
    ((content_length != -1) && (MARK_LEN < content_length))
  }

  action consume_octet {
    ((content_length == -1) || (MARK_LEN < content_length))
  }

  action check_message_size {
    message_size += 1;
    //raise MessageSizeExceeded if message_size > max_message_size
  }

  action finish_message {
    rb_yield(message);
    message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

typedef struct {
  long long max_message_size;

  const char *chunk;
  const char *p;
  int cs;
  size_t mark;
  const char *mark_key;
  VALUE mark_message;
  size_t mark_message_size;
  size_t mark_message_content_length;
} parser_state_t;

VALUE cMessage = Qnil;

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->mark_message);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  state->mark_message = Qtrue;
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  VALUE max_message_size;
  // rb_scan_args(argc, argv, "01", &max_message_size);
}

static VALUE parser_message(VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);
  return state->mark_message;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));
  VALUE mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "state_check", parser_message, 0);
}


Fixed naming of the FormattedTextTokenizer machine

package com.readytalk.swt.text.tokenizer;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.nio.charset.Charset;

import org.eclipse.swt.SWT;

import com.readytalk.swt.text.painter.TextType;

/*
 * This parser will automatically build via the gradle build process.  If you want to
 * rebuild this parser by hand, run the following command:
 *
 *    ragel -J WikiTextTokenizer.java.rl -o WikiTextTokenizer.java
 */
public class FormattedTextTokenizer implements TextTokenizer {

	private Charset encoding = Charset.defaultCharset();
	private List<TextToken> tokens = new ArrayList<TextToken>();

	private int styleState = 0x00;

	@Override
	public FormattedTextTokenizer setEncoding(final Charset encoding) {
		this.encoding = encoding;
		return this;
	}

	@Override
	public Charset getEncoding() {
		return encoding;
	}

	@Override
	public FormattedTextTokenizer reset() {
	  tokens.clear();
	  return this;
	}

    void emit(TextType type, final byte[] data, final int start, final int end) {
      String text = spliceToString(data, start, end);
      if (TextType.WHITESPACE.equals(type)) {
        if("\n".equals(text)) {
          type = TextType.NEWLINE;
        }
      }
      tokens.add(new TextToken(type, text));
    }

    byte[] splice(final byte[] data, final int start, final int end) {
      int length = end-start;
      byte[] splicedData = new byte[length];
      System.arraycopy(data, start, splicedData, 0, length);
      return splicedData;
    }

    String spliceToString(final byte[] data, final int start, final int end) {
      return new String(splice(data, start, end), encoding);
    }

    @Override
    public List<TextToken> tokenize(final String text) {

      if(text == null || "".equals(text)) {
        return tokens;
      }

      byte[] data = text.getBytes(encoding);
      int eof = data.length;

      // the names of these variables are specified by the ragel parser generator
      int p = 0, pe = data.length, te, ts, cs, act;

      %%{
      machine FormattedTextScanner;
      part              = (any - (space|'\''))+;
    	word              = part ('\''|part)*;
    	url               = ('http'|'https'|'file') '://' (any - space)+;
    	link              = '[' url ((' '|'\t')+ word* )? ']';
    	boldAndItalicText = '\'\'\'\'\'';
    	boldText          = '\'\'\'';
    	italicText        = '\'\'';

    	main := |*
    	  link              => { scanLink(splice(data, ts, te)); };
    	  url               => {
    		  String url = spliceToString(data, ts, te);
    		  try {
    			  tokens.add(new TextToken(TextType.NAKED_URL, url).setUrl(new URL(url)));
			    } catch (MalformedURLException exception) {
				    tokens.add(new TextToken(TextType.TEXT, text));
			    }
    	  };
    	  word             => {
          switch(styleState) {
            case SWT.BOLD:
              emit(TextType.BOLD, data, ts, te);
              break;
            case SWT.ITALIC:
              emit(TextType.ITALIC, data, ts, te);
              break;
            case SWT.BOLD|SWT.ITALIC:
              emit(TextType.BOLD_AND_ITALIC, data, ts, te);
              break;
            default:
              emit(TextType.TEXT, data, ts, te);
          }
    	  };
    	  boldAndItalicText => { styleState ^= SWT.BOLD|SWT.ITALIC; };
    	  boldText          => { styleState ^= SWT.BOLD; };
    	  italicText        => { styleState ^= SWT.ITALIC; };
    	  space             => { emit(TextType.WHITESPACE, data, ts, te); };
    	*|;

    	write init;
    	write exec;
      }%%

      return tokens;
    }
    %% write data;

    private void scanLink(byte[] data) {
      int eof = data.length;
      int p = 0, pe = data.length, te, ts, cs, act;
      String url = "";
      String text = "";
      %%{
	    machine URLParser;
		lbrace   = '[';
		rbrace   = ']';
		url      = (' ')* ('http'|'https'|'file') '://' (any - (' '|'\t'| '[' | ']'))+;
		linkText = (any - (' ' | '\n' | '[' | ']'))+;

		main := |*
		  lbrace;
		  rbrace;
		  url        => { url  += spliceToString(data, ts, te); };
		  linkText   => { text += spliceToString(data, ts, te); };
		  space      => { text += spliceToString(data, ts, te); };
		*|;

		write init;
		write exec;
	  }%%

	  try {
		if (text == null || "".equals(text)){
          tokens.add(new TextToken(TextType.LINK_URL, url).setUrl(new URL(url)));
		} else {
          tokens.add(new TextToken(TextType.LINK_AND_NAMED_URL, text).setUrl(new URL(url)));
		}
	  } catch (MalformedURLException exception) {
		tokens.add(new TextToken(TextType.TEXT, text + " (" + url + ") "));
	  }
    }
    %% write data;
}

package com.readytalk.swt.text.tokenizer;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.nio.charset.Charset;

import org.eclipse.swt.SWT;

import com.readytalk.swt.text.painter.TextType;

/*
 * This parser will automatically build via the gradle build process.  If you want to
 * rebuild this parser by hand, run the following command:
 *
 *    ragel -J WikiTextTokenizer.java.rl -o WikiTextTokenizer.java
 */
public class FormattedTextTokenizer implements TextTokenizer {

	private Charset encoding = Charset.defaultCharset();
	private List<TextToken> tokens = new ArrayList<TextToken>();

	private int styleState = 0x00;

	@Override
	public FormattedTextTokenizer setEncoding(final Charset encoding) {
		this.encoding = encoding;
		return this;
	}

	@Override
	public Charset getEncoding() {
		return encoding;
	}

	@Override
	public FormattedTextTokenizer reset() {
	  tokens.clear();
	  return this;
	}

    void emit(TextType type, final byte[] data, final int start, final int end) {
      String text = spliceToString(data, start, end);
      if (TextType.WHITESPACE.equals(type)) {
        if("\n".equals(text)) {
          type = TextType.NEWLINE;
        }
      }
      tokens.add(new TextToken(type, text));
    }

    byte[] splice(final byte[] data, final int start, final int end) {
      int length = end-start;
      byte[] splicedData = new byte[length];
      System.arraycopy(data, start, splicedData, 0, length);
      return splicedData;
    }

    String spliceToString(final byte[] data, final int start, final int end) {
      return new String(splice(data, start, end), encoding);
    }

    @Override
    public List<TextToken> tokenize(final String text) {

      if(text == null || "".equals(text)) {
        return tokens;
      }

      byte[] data = text.getBytes(encoding);
      int eof = data.length;

      // the names of these variables are specified by the ragel parser generator
      int p = 0, pe = data.length, te, ts, cs, act;

      %%{
      machine WikiTextScanner;
      part              = (any - (space|'\''))+;
    	word              = part ('\''|part)*;
    	url               = ('http'|'https'|'file') '://' (any - space)+;
    	link              = '[' url ((' '|'\t')+ word* )? ']';
    	boldAndItalicText = '\'\'\'\'\'';
    	boldText          = '\'\'\'';
    	italicText        = '\'\'';

    	main := |*
    	  link              => { scanLink(splice(data, ts, te)); };
    	  url               => {
    		  String url = spliceToString(data, ts, te);
    		  try {
    			  tokens.add(new TextToken(TextType.NAKED_URL, url).setUrl(new URL(url)));
			    } catch (MalformedURLException exception) {
				    tokens.add(new TextToken(TextType.TEXT, text));
			    }
    	  };
    	  word             => {
          switch(styleState) {
            case SWT.BOLD:
              emit(TextType.BOLD, data, ts, te);
              break;
            case SWT.ITALIC:
              emit(TextType.ITALIC, data, ts, te);
              break;
            case SWT.BOLD|SWT.ITALIC:
              emit(TextType.BOLD_AND_ITALIC, data, ts, te);
              break;
            default:
              emit(TextType.TEXT, data, ts, te);
          }
    	  };
    	  boldAndItalicText => { styleState ^= SWT.BOLD|SWT.ITALIC; };
    	  boldText          => { styleState ^= SWT.BOLD; };
    	  italicText        => { styleState ^= SWT.ITALIC; };
    	  space             => { emit(TextType.WHITESPACE, data, ts, te); };
    	*|;

    	write init;
    	write exec;
      }%%

      return tokens;
    }
    %% write data;

    private void scanLink(byte[] data) {
      int eof = data.length;
      int p = 0, pe = data.length, te, ts, cs, act;
      String url = "";
      String text = "";
      %%{
	    machine URLParser;
		lbrace   = '[';
		rbrace   = ']';
		url      = (' ')* ('http'|'https'|'file') '://' (any - (' '|'\t'| '[' | ']'))+;
		linkText = (any - (' ' | '\n' | '[' | ']'))+;

		main := |*
		  lbrace;
		  rbrace;
		  url        => { url  += spliceToString(data, ts, te); };
		  linkText   => { text += spliceToString(data, ts, te); };
		  space      => { text += spliceToString(data, ts, te); };
		*|;

		write init;
		write exec;
	  }%%

	  try {
		if (text == null || "".equals(text)){
          tokens.add(new TextToken(TextType.LINK_URL, url).setUrl(new URL(url)));
		} else {
          tokens.add(new TextToken(TextType.LINK_AND_NAMED_URL, text).setUrl(new URL(url)));
		}
	  } catch (MalformedURLException exception) {
		tokens.add(new TextToken(TextType.TEXT, text + " (" + url + ") "));
	  }
    }
    %% write data;
}


Remove unused ragel file.

Remove unused ragel file.

Todo: Learn ragel, it seems cool

#include <string.h>
#include <stdlib.h>
#include <lemon.h>

// does this make me a bad person?
#ifdef DEBUG_PRINT
#define S_RET(it) 	fprintf(stderr, "{%s}", #it); return it
#define S_RETT(it) 	fprintf(stderr, "{%s[%s]}", #it, yytext); return it
#else
#define S_RET(it)	return it
#define S_RETT(it)	return it
#endif

void token(int tok) {
	char *data = ts;
	int len = te-ts;
	fprintf(stderr, "<%d>", tok);
}

%%{

	machine pascal_lexer;

	digits = digit digit*;
	optional_fraction = '.' digits | empty;
	optional_exponent = ( 'E' ('+' | '-' | empty) digits | empty;
	num = digits optional_fraction optional_exponent;

	main := |*
		'program' =>    { S_RET(PROGRAM); };
		'array' =>      { S_RET(ARRAY); };
		'of' =>         { S_RET(OF); };
		'integer' =>    { S_RET(INTEGER); };
		'real' =>       { S_RET(REAL); };
		'function' =>   { S_RET(FUNCTION); };
		'procedure' =>  { S_RET(PROCEDURE); };
		'begin' =>      { S_RET(BEGIN); };
		'end' =>        { S_RET(END); };
		'if' =>         { S_RET(IF); };
		'then' =>       { S_RET(THEN); };
		'else' =>       { S_RET(ELSE); };
		'while' =>      { S_RET(WHILE); };
		'do' =>         { S_RET(DO); };
		'not' =>        { S_RET(NOT); };
		'var' =>        { S_RET(VAR); };

		'..' =>         { S_RET(DOTDOT); };
		':=' =>         { S_RET(ASSIGNOP); };

		'(' =>          { S_RET(LPAREN); };
		')' =>          { S_RET(RPAREN); };
		'[' =>          { S_RET(RBRKT); };
		']' =>          { S_RET(LBRKT); };

		'=' =>          { S_RETT(RELOP); };
		'<>' =>         { S_RETT(RELOP); };
		'<' =>          { S_RETT(RELOP); };
		'>' =>          { S_RETT(RELOP); };
		'<=' =>         { S_RETT(RELOP); };
		'>=' =>         { S_RETT(RELOP); };

		'+' =>          { S_RETT(ADDOP); };
		'-' =>          { S_RETT(ADDOP); };
		'or' =>         { S_RETT(ADDOP); };

		'and' =>        { S_RETT(MULOP); };
		'*' =>          { S_RETT(MULOP); };
		'/' =>          { S_RETT(MULOP); };
		'div' =>        { S_RETT(MULOP); };
		'mod' =>        { S_RETT(MULOP); };
	*|;

}%%