fod_lexer.rl: suppress '-Wswitch-default' and '-Wsign-conversion'

Signed-off-by: Pavel Kretov <7cae407f1f3b56350827d1394be40a3e8f9cb303@gmail.com>

#include "fod_common.h"
#include "fod_lexer.h"
#include <string.h>
#include <stdio.h>

%% machine fod_lexer;
%% write data;

void fod_lexer_reset(
	struct fod_lexer *lex,
	char const       *input,
	fod_reallocator   realloc,
	void             *realloc_arg) {

    lex->realloc             = realloc;
    lex->realloc_arg         = realloc_arg;
    lex->input               = input;
    lex->input_end           = input + strlen(input);
    lex->is_eof              = 0;
    lex->is_error            = 0;
    lex->current_token_start = NULL;
    lex->current_token_end   = NULL;
    lex->current_position    = input;
    lex->current_state       = fod_lexer_start;
    lex->current_action      = 0;
}

int fod_lexer_tokenize(
        struct fod_lexer        *lex,
        enum fod_lexeme_major   *out_major,
        struct fod_lexeme_minor *out_minor) {

    /* Don't use ragel's 'write init;' construct. Instead, perform manual
       state restoration. */
    char const *p   = lex->current_position;
    char const *pe  = lex->input_end;
    char const *ts  = lex->current_token_start;
    char const *te  = lex->current_token_end;
    char const *eof = pe;

    int cs  = lex->current_state;
    int act = lex->current_action;

    /* In Lemon, valid token major code is never zero, 'cause zero
       means 'EOF'. Let's return it in case of errors. */
    *out_major = 0;

    #pragma GCC diagnostic push
    #pragma GCC diagnostic ignored "-Wswitch-default"
    #pragma GCC diagnostic ignored "-Wsign-conversion"

    %%{
    number = digit+;
    single_quoted_string = "'" ([^']|"\\'")* "'";
    double_quoted_string = '"' ([^"]|'\\"')* '"';
    main := |*
        # Skip any whitespaces which are not in string literal.
        space+ => { printf("SPACES\n"); };

        # Parse
        '=='  => { *out_major = LEX_OPERATOR_EQ; printf("==\n"); fbreak; };
        '!='  => { *out_major = LEX_OPERATOR_NE; printf("!=\n"); fbreak; };
        '<='  => { *out_major = LEX_OPERATOR_LE; printf("<=\n"); fbreak; };
        '>='  => { *out_major = LEX_OPERATOR_GE; printf(">=\n"); fbreak; };
        '<'   => { *out_major = LEX_OPERATOR_LT; printf("<\n");  fbreak; };
        '>'   => { *out_major = LEX_OPERATOR_GT; printf(">\n");  fbreak; };
        '('   => { *out_major = LEX_PAREN_LEFT;  printf("(\n");  fbreak; };
        ')'   => { *out_major = LEX_PAREN_RIGHT; printf(")\n");  fbreak; };

        #
        '&&'  => { *out_major = LEX_OPERATOR_AND; printf("&&\n"); fbreak; };
        '||'  => { *out_major = LEX_OPERATOR_OR;  printf("||\n"); fbreak; };
        '!'   => { *out_major = LEX_OPERATOR_NOT; printf("!\n");  fbreak; };

        # Unsigned integer literal.
        number => {
	    fod_longest_uint res;
	    if (!fod_substring_to_uint(&res, ts, te))
	       fbreak;

            *out_major = LEX_LITERAL_NUMBER;
            out_minor->literal_number = res;
            fbreak;
        };

        single_quoted_string |
        double_quoted_string => {
	    char *str = fod_substring_duplicate_and_unquote(ts, te,
                                            lex->realloc, lex->realloc_arg);
	    if (str == NULL)
	        fbreak;

            *out_major = LEX_LITERAL_STRING;
            out_minor->literal_string = str;
            fbreak;
        };

        # Parameters returning cl_bool.

        'cl_device_available'
	| 'device_available'
	| 'deviceAvailable'
	| 'available' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_AVAILABLE;
	        fbreak;
        };

        'cl_device_compiler_available'
	| 'device_compiler_available'
	| 'deviceCompilerAvailable'
	| 'compiler_available'
	| 'compilerAvailable' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_COMPILER_AVAILABLE;
	        fbreak;
        };

        'cl_device_endian_little'
	| 'device_endian_little'
	| 'endian_little'
	| 'little_endian'
	| 'deviceEndianLittle'
	| 'deviceLittleEndian'
	| 'endianLittle'
	| 'littleEndian' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ENDIAN_LITTLE;
	        fbreak;
        };

	'cl_device_error_correction_support'
	| 'device_error_correction_support'
	| 'device_error_correction'
	| 'error_correction_support'
	| 'error_correction'
	| 'deviceErrorCorrectionSupport'
	| 'deviceErrorCorrection'
	| 'errorCorrectionSupport'
	| 'errorCorrection' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ERROR_CORRECTION_SUPPORT;
	        fbreak;
	};

	'cl_exec_kernel'
	| 'exec_kernel'
	| 'device_exec_kernel'
	| 'execKernel'
	| 'deviceExecKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_KERNEL;
	        fbreak;
        };

	'cl_exec_native_kernel'
	| 'exec_native_kernel'
	| 'device_exec_native_kernel'
	| 'execNativeKernel'
	| 'deviceExecNativeKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_NATIVE_KERNEL;
	        fbreak;
        };

	'cl_device_image_support'
	| 'device_image_support'
	| 'image_support'
	| 'deviceImageSupport'
	| 'imageSupport' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_IMAGE_SUPPORT;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_address_bits'
	| 'device_address_bits'
	| 'address_bits'
	| 'deviceAddressBits'
	| 'addressBits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_ADDRESS_BITS;
	        fbreak;
        };

	'cl_device_global_mem_cacheline_size'
	| 'device_global_mem_cacheline_size'
	| 'global_mem_cacheline_size'
	| 'deviceglobalmemcachelinesize'
	| 'globalmemcachelinesize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE;
	        fbreak;
        };

	'cl_device_max_clock_frequency'
	| 'device_max_clock_frequency'
	| 'max_clock_frequency'
	| 'deviceMaxClockFrequency'
	| 'maxClockFrequency'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CLOCK_FREQUENCY;
	        fbreak;
        };

	'cl_device_max_compute_units'
	| 'device_max_compute_units'
	| 'max_compute_units'
	| 'deviceMaxComputeUnits'
	| 'maxComputeUnits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_COMPUTE_UNITS;
	        fbreak;
        };

	'cl_device_max_constant_args'
	| 'device_max_constant_args'
	| 'max_constant_args'
	| 'deviceMaxConstantArgs'
	| 'maxConstantArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_ARGS;
	        fbreak;
        };

	'cl_device_max_read_image_args'
	| 'device_max_read_image_args'
	| 'max_read_image_args'
	| 'deviceMaxReadImageArgs'
	| 'maxReadImageArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_READ_IMAGE_ARGS;
	        fbreak;
        };

	'cl_device_global_mem_size'
	| 'device_global_mem_size'
	| 'global_mem_size'
	| 'deviceGlobalMemSize'
	| 'globalMemSize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_SIZE;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_local_mem_size'
	| 'device_local_mem_size'
	| 'local_mem_size'
	| 'deviceLocalMemSize'
	| 'localMemSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_LOCAL_MEM_SIZE;
	        fbreak;
        };

	'cl_device_max_constant_buffer_size'
	| 'device_max_constant_buffer_size'
	| 'max_constant_buffer_size'
	| 'deviceMaxConstantBufferSize'
	| 'maxConstantBufferSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE;
	        fbreak;
        };

	'cl_device_max_mem_alloc_size'
	| 'device_max_mem_alloc_size'
	| 'max_mem_alloc_size'
	| 'deviceMaxMemAllocSize'
	| 'maxMemAllocSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_MEM_ALLOC_SIZE;
	        fbreak;
        };

	'cl_device_global_mem_cache_size'
	| 'device_global_mem_cache_size'
	| 'global_mem_cache_size'
	| 'deviceGlobalMemCacheSize'
	| 'globalMemCacheSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHE_SIZE;
	        fbreak;
        };

	# Parameters returning size_t.

	# Parameters returning string.

    *|;

    write exec;

    }%%

    #pragma GCC diagnostic pop

    if (*out_major == 0) {
        if (p == pe)
	    lex->is_eof = 1;
	else
	    lex->is_error = 1;

        return 0;
    }

    /* No need to save 'pe' here. */
    lex->current_state       = cs;
    lex->current_position    = p;
    lex->current_token_start = ts;
    lex->current_token_end   = te;
    lex->current_action      = act;
    return 1;
}

#include "fod_common.h"
#include "fod_lexer.h"
#include <string.h>
#include <stdio.h>

%% machine fod_lexer;
%% write data;

void fod_lexer_reset(
	struct fod_lexer *lex,
	char const       *input,
	fod_reallocator   realloc,
	void             *realloc_arg) {

    lex->realloc             = realloc;
    lex->realloc_arg         = realloc_arg;
    lex->input               = input;
    lex->input_end           = input + strlen(input);
    lex->is_eof              = 0;
    lex->is_error            = 0;
    lex->current_token_start = NULL;
    lex->current_token_end   = NULL;
    lex->current_position    = input;
    lex->current_state       = fod_lexer_start;
    lex->current_action      = 0;
}

int fod_lexer_tokenize(
        struct fod_lexer        *lex,
        enum fod_lexeme_major   *out_major,
        struct fod_lexeme_minor *out_minor) {

    /* Don't use ragel's 'write init;' construct. Instead, perform manual
       state restoration. */
    char const *p   = lex->current_position;
    char const *pe  = lex->input_end;
    char const *ts  = lex->current_token_start;
    char const *te  = lex->current_token_end;
    char const *eof = pe;

    int cs  = lex->current_state;
    int act = lex->current_action;

    /* In Lemon, valid token major code is never zero, 'cause zero
       means 'EOF'. Let's return it in case of errors. */
    *out_major = 0;

    %%{
    number = digit+;
    single_quoted_string = "'" ([^']|"\\'")* "'";
    double_quoted_string = '"' ([^"]|'\\"')* '"';
    main := |*
        # Skip any whitespaces which are not in string literal.
        space+ => { printf("SPACES\n"); };

        # Parse
        '=='  => { *out_major = LEX_OPERATOR_EQ; printf("==\n"); fbreak; };
        '!='  => { *out_major = LEX_OPERATOR_NE; printf("!=\n"); fbreak; };
        '<='  => { *out_major = LEX_OPERATOR_LE; printf("<=\n"); fbreak; };
        '>='  => { *out_major = LEX_OPERATOR_GE; printf(">=\n"); fbreak; };
        '<'   => { *out_major = LEX_OPERATOR_LT; printf("<\n");  fbreak; };
        '>'   => { *out_major = LEX_OPERATOR_GT; printf(">\n");  fbreak; };
        '('   => { *out_major = LEX_PAREN_LEFT;  printf("(\n");  fbreak; };
        ')'   => { *out_major = LEX_PAREN_RIGHT; printf(")\n");  fbreak; };

        #
        '&&'  => { *out_major = LEX_OPERATOR_AND; printf("&&\n"); fbreak; };
        '||'  => { *out_major = LEX_OPERATOR_OR;  printf("||\n"); fbreak; };
        '!'   => { *out_major = LEX_OPERATOR_NOT; printf("!\n");  fbreak; };

        # Unsigned integer literal.
        number => {
	    fod_longest_uint res;
	    if (!fod_substring_to_uint(&res, ts, te))
	       fbreak;

            *out_major = LEX_LITERAL_NUMBER;
            out_minor->literal_number = res;
            fbreak;
        };

        single_quoted_string |
        double_quoted_string => {
	    char *str = fod_substring_duplicate_and_unquote(ts, te,
                                            lex->realloc, lex->realloc_arg);
	    if (str == NULL)
	        fbreak;

            *out_major = LEX_LITERAL_STRING;
            out_minor->literal_string = str;
            fbreak;
        };

        # Parameters returning cl_bool.

        'cl_device_available'
	| 'device_available'
	| 'deviceAvailable'
	| 'available' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_AVAILABLE;
	        fbreak;
        };

        'cl_device_compiler_available'
	| 'device_compiler_available'
	| 'deviceCompilerAvailable'
	| 'compiler_available'
	| 'compilerAvailable' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_COMPILER_AVAILABLE;
	        fbreak;
        };

        'cl_device_endian_little'
	| 'device_endian_little'
	| 'endian_little'
	| 'little_endian'
	| 'deviceEndianLittle'
	| 'deviceLittleEndian'
	| 'endianLittle'
	| 'littleEndian' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ENDIAN_LITTLE;
	        fbreak;
        };

	'cl_device_error_correction_support'
	| 'device_error_correction_support'
	| 'device_error_correction'
	| 'error_correction_support'
	| 'error_correction'
	| 'deviceErrorCorrectionSupport'
	| 'deviceErrorCorrection'
	| 'errorCorrectionSupport'
	| 'errorCorrection' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ERROR_CORRECTION_SUPPORT;
	        fbreak;
	};

	'cl_exec_kernel'
	| 'exec_kernel'
	| 'device_exec_kernel'
	| 'execKernel'
	| 'deviceExecKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_KERNEL;
	        fbreak;
        };

	'cl_exec_native_kernel'
	| 'exec_native_kernel'
	| 'device_exec_native_kernel'
	| 'execNativeKernel'
	| 'deviceExecNativeKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_NATIVE_KERNEL;
	        fbreak;
        };

	'cl_device_image_support'
	| 'device_image_support'
	| 'image_support'
	| 'deviceImageSupport'
	| 'imageSupport' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_IMAGE_SUPPORT;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_address_bits'
	| 'device_address_bits'
	| 'address_bits'
	| 'deviceAddressBits'
	| 'addressBits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_ADDRESS_BITS;
	        fbreak;
        };

	'cl_device_global_mem_cacheline_size'
	| 'device_global_mem_cacheline_size'
	| 'global_mem_cacheline_size'
	| 'deviceglobalmemcachelinesize'
	| 'globalmemcachelinesize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE;
	        fbreak;
        };

	'cl_device_max_clock_frequency'
	| 'device_max_clock_frequency'
	| 'max_clock_frequency'
	| 'deviceMaxClockFrequency'
	| 'maxClockFrequency'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CLOCK_FREQUENCY;
	        fbreak;
        };

	'cl_device_max_compute_units'
	| 'device_max_compute_units'
	| 'max_compute_units'
	| 'deviceMaxComputeUnits'
	| 'maxComputeUnits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_COMPUTE_UNITS;
	        fbreak;
        };

	'cl_device_max_constant_args'
	| 'device_max_constant_args'
	| 'max_constant_args'
	| 'deviceMaxConstantArgs'
	| 'maxConstantArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_ARGS;
	        fbreak;
        };

	'cl_device_max_read_image_args'
	| 'device_max_read_image_args'
	| 'max_read_image_args'
	| 'deviceMaxReadImageArgs'
	| 'maxReadImageArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_READ_IMAGE_ARGS;
	        fbreak;
        };

	'cl_device_global_mem_size'
	| 'device_global_mem_size'
	| 'global_mem_size'
	| 'deviceGlobalMemSize'
	| 'globalMemSize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_SIZE;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_local_mem_size'
	| 'device_local_mem_size'
	| 'local_mem_size'
	| 'deviceLocalMemSize'
	| 'localMemSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_LOCAL_MEM_SIZE;
	        fbreak;
        };

	'cl_device_max_constant_buffer_size'
	| 'device_max_constant_buffer_size'
	| 'max_constant_buffer_size'
	| 'deviceMaxConstantBufferSize'
	| 'maxConstantBufferSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE;
	        fbreak;
        };

	'cl_device_max_mem_alloc_size'
	| 'device_max_mem_alloc_size'
	| 'max_mem_alloc_size'
	| 'deviceMaxMemAllocSize'
	| 'maxMemAllocSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_MEM_ALLOC_SIZE;
	        fbreak;
        };

	'cl_device_global_mem_cache_size'
	| 'device_global_mem_cache_size'
	| 'global_mem_cache_size'
	| 'deviceGlobalMemCacheSize'
	| 'globalMemCacheSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHE_SIZE;
	        fbreak;
        };

	# Parameters returning size_t.

	# Parameters returning string.

    *|;

    write exec;

    }%%

    if (*out_major == 0) {
        if (p == pe)
	    lex->is_eof = 1;
	else
	    lex->is_error = 1;

        return 0;
    }

    /* No need to save 'pe' here. */
    lex->current_state       = cs;
    lex->current_position    = p;
    lex->current_token_start = ts;
    lex->current_token_end   = te;
    lex->current_action      = act;
    return 1;
}


Fix buffer concatenation to work on ruby 1.9.3

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  delimiter        = 0;
  cr               = "\r";
  lf               = "\n";
  separator        = ' ' | cr lf | lf;
  quote            = "'";
  double_quote     = '"';
  underscore       = '_';

  quoted_string    = quote (^quote | quote{2})** quote;

  unsigned_integer = digit+;

  identifier_start = alpha;
  identifier_part  = identifier_start | digit | double_quote{2};
  identifier_body  = identifier_start (underscore? identifier_part)*;
  identifier       = double_quote identifier_body double_quote;

  main := |*
    delimiter;
    separator;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    '%'              { emit('mod');         };
    '^'              { emit('pow');         };
    'E'              { emit('E');           };

    quoted_string    { emit_string();           };
    unsigned_integer { emit_unsigned_integer(); };

    identifier       { emit_identifier(); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE  = 1024
    CHUNK_SIZE  = 1  # XXX: used for testing
    END_OF_FILE = [0].freeze

    %% write data;

    def initialize(input)
      @input    = input
      @encoding = @input.external_encoding
      @block    = nil
      @buffer   = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      begin
        chunk = read_chunk
        @buffer.push(*chunk)
        %% write exec;
        optimize_buffer
      end until chunk == END_OF_FILE
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_identifier
      identifier = text
      identifier.sub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_string
      string = text
      string.sub!(/\A'(.*)'\z/, '\1') and string.gsub!("''", "'")
      emit(:string, string)
    end

    def text
      @buffer[@ts...@te].pack('C*').force_encoding(@encoding)
    end

    def read_chunk
      @input.read_nonblock(CHUNK_SIZE).bytes
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      END_OF_FILE
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  delimiter        = 0;
  cr               = "\r";
  lf               = "\n";
  separator        = ' ' | cr lf | lf;
  quote            = "'";
  double_quote     = '"';
  underscore       = '_';

  quoted_string    = quote (^quote | quote{2})** quote;

  unsigned_integer = digit+;

  identifier_start = alpha;
  identifier_part  = identifier_start | digit | double_quote{2};
  identifier_body  = identifier_start (underscore? identifier_part)*;
  identifier       = double_quote identifier_body double_quote;

  main := |*
    delimiter;
    separator;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    '%'              { emit('mod');         };
    '^'              { emit('pow');         };
    'E'              { emit('E');           };

    quoted_string    { emit_string();           };
    unsigned_integer { emit_unsigned_integer(); };

    identifier       { emit_identifier(); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE  = 1024
    CHUNK_SIZE  = 1  # XXX: used for testing
    END_OF_FILE = [0].freeze

    %% write data;

    def initialize(input)
      @input    = input
      @encoding = @input.external_encoding
      @block    = nil
      @buffer   = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      begin
        chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end until chunk == END_OF_FILE
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_identifier
      identifier = text
      identifier.sub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_string
      string = text
      string.sub!(/\A'(.*)'\z/, '\1') and string.gsub!("''", "'")
      emit(:string, string)
    end

    def text
      @buffer[@ts...@te].pack('C*').force_encoding(@encoding)
    end

    def read_chunk
      @input.read_nonblock(CHUNK_SIZE).bytes
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      END_OF_FILE
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL


create bel_token pointers for each scanner token

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-ast.h"
#include "bel-node-stack.h"
#include "bel-parser.h"

typedef enum {
    IDENT,
    STRING,
    O_PAREN,
    C_PAREN,
    COLON,
    COMMA,
    SPACES
} bel_token_type;

typedef struct {
    bel_token_type type;
    char           *value;
} bel_token;

bel_token* bel_new_token(bel_token_type type, char* input, int ts, int te) {
    bel_token *new_token;
    int       length;
    char      *copy_input;

    new_token = malloc(sizeof(bel_token));
    new_token->type = type;

    length = te - ts;
    copy_input = malloc(length + 1);
    strncpy(copy_input, input, length);

    new_token->value = copy_input;
    return new_token;
};

%%{
    machine set;
    write data;
}%%

bel_token* parse_term_lookahead(char* line) {
    int             cs;
    char            *p;
    char            *pe;
    int             ts;
    int             te;
    int             act;
    char            *eof;
    int             top;
    char            *value;
    int             fi;
    int             vi;
    bel_token       *tokens[256];

    p            = line;
    pe           = line + strlen(line);
    eof          = pe;
    top          = 0;
    value        = malloc(sizeof(char) * VALUE_CHAR_LEN);
    fi           = 0;
    vi           = 0;

    memset(value, '\0', VALUE_CHAR_LEN);

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"';

        action IDENT   { bel_new_token(IDENT, p, ts, te);   }
        action STRING  { bel_new_token(STRING, p, ts, te);  }
        action O_PAREN { bel_new_token(O_PAREN, p, ts, te); }
        action C_PAREN { bel_new_token(C_PAREN, p, ts, te); }
        action COLON   { bel_new_token(COLON, p, ts, te);   }
        action COMMA   { bel_new_token(COMMA, p, ts, te);   }
        action SPACES  { bel_new_token(SPACES, p, ts, te);  }

        term := |*
            IDENT    => IDENT;
            STRING   => STRING;
            O_PAREN  => O_PAREN;
            C_PAREN  => C_PAREN;
            COLON    => COLON;
            COMMA    => COMMA;
            SPACES   => SPACES;
        *|;

        # Initialize and execute.
        write init;
        write exec;
    }%%

    free(value);

    return tokens;
};
// vim: ft=c sw=4 ts=4 sts=4 expandtab

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-ast.h"
#include "bel-node-stack.h"
#include "bel-parser.h"

typedef enum {
    IDENT,
    STRING,
    O_PAREN,
    C_PAREN,
    COLON,
    COMMA,
    SPACES
} bel_token_type;

typedef struct {
    bel_token_type type;
    char           *value;
} bel_token;

%%{
    machine set;
    write data;
}%%

bel_token* parse_term_lookahead(char* line) {
    int             cs;
    char            *p;
    char            *pe;
    int             ts;
    int             te;
    int             act;
    char            *eof;
    int             top;
    char            *value;
    int             fi;
    int             vi;
    bel_token       *tokens;

    p            = line;
    pe           = line + strlen(line);
    eof          = pe;
    top          = 0;
    value        = malloc(sizeof(char) * VALUE_CHAR_LEN);
    tokens       = malloc(sizeof(bel_token) * 256);
    fi           = 0;
    vi           = 0;

    memset(value, '\0', VALUE_CHAR_LEN);

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"';

        action STATE {
            fprintf(stdout, "state: %d\n", fcurs);
        }

        term := |*
            IDENT    => { fprintf(stdout, "IDENT\n");   };
            STRING   => { fprintf(stdout, "STRING\n");  };
            O_PAREN  => { fprintf(stdout, "O_PAREN\n"); };
            C_PAREN  => { fprintf(stdout, "C_PAREN\n"); };
            COLON    => { fprintf(stdout, "COLON\n");   };
            COMMA    => { fprintf(stdout, "COMMA\n");   };
            SPACES   => { fprintf(stdout, "SPACES\n");  };
        *|;

        # Initialize and execute.
        write init;
        write exec;
    }%%

    free(value);

    return tokens;
};
// vim: ft=c sw=4 ts=4 sts=4 expandtab


Limited validation of ID field

Limited validation of ID field

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  key_value = ( id_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:key_word,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A' ) >mark %{ emit.call(:value,data,ts,p) } );

  id        = ( 'ID=' id_value ) @kw;
  key_value = ( (id | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Removed unnecessary dependency from Lua Scanner

package net.ohloh.ohcount4j.scan;

import net.ohloh.ohcount4j.Language;

public class LuaScanner extends BaseScanner{

  %%{
    machine lua;
    include common "common.rl";

    lua_line_comment = '--' @comment (nonnewline? @comment |
    	(([^\[] nonnewline) | (nonnewline [^\[])) @comment nonnewline* @comment);

    lua_block_comment_begin = '--[[' @comment;
    lua_block_comment_end = ']]' @comment;

    lua_block_comment := |*
	    lua_block_comment_end => { fret; };
		spaces;
		newline;
		(any - newline) => comment;
    *|;

  	lua_line := |*
		lua_block_comment_begin => { fcall lua_block_comment; };
    	lua_line_comment;
    	spaces;
    	string_literal;
    	newline;
    	(any - newline) => code;
  	*|;
  }%%

  %% write data;

  @Override
  public void doScan(){
	// variables and data is set up in BaseScanner
    %% write init;
    init();
    %% write exec;
  }

  @Override
  public Language getLanguage(){
  	return Language.LANG_LUA;
  }

}

package net.ohloh.ohcount4j.scan;

import net.ohloh.ohcount4j.Language;

public class LuaScanner extends BaseScanner{

  %%{
    machine lua;
    include common "common.rl";
    include c "c.rl";

    lua_line_comment = '--' @comment (nonnewline? @comment
    	| (([^\[] nonnewline) | (nonnewline [^\[])) @comment nonnewline* @comment);

    lua_block_comment_begin = '--[[' @comment;
    lua_block_comment_end = ']]' @comment;

    lua_block_comment := |*
	    lua_block_comment_end => { fret; };
		spaces;
		newline;
		(any - newline) => comment;
    *|;

  	lua_line := |*
		lua_block_comment_begin => { fcall lua_block_comment; };
    	lua_line_comment;
    	spaces;
    	string_literal;
    	newline;
    	(any - newline) => code;
  	*|;
  }%%

  %% write data;

  @Override
  public void doScan(){
	// variables and data is set up in BaseScanner
    %% write init;
    init();
    %% write exec;
  }

  @Override
  public Language getLanguage(){
  	return Language.LANG_LUA;
  }

}


moved some codeparts within the lexer

moved some codeparts within the lexer

%%machine lexer;

module CakeLang
  class Lexer
    %% write data;

    CONVERSION = {
        :T_INT => :to_i,
        :T_FLOAT => :to_f
      }

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
    end

    # @ts, @te -> token start, token end
    # had to use @te-1 for token range end because 1 digit token (eg.: ',(,),+) and no whitespace between will
    # break the tokenization
    def data(start = @ts, stop = (@te-1))
      return @data[start..stop].pack('U*')
    end

    def emit(symbol, start = @ts, stop = (@te-1))
      value = data(start, stop)

      if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
        value = value.send(CONVERSION[symbol])
      end

      @tokens << [symbol, value]
    end

    def lex(data)
      @data = data.unpack("U*")
      lexer_start = self.class.lexer_start
      eof = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    %%{
      access @;

      T_KEYWORD_DEF = '->';
      T_KEYWORD_END = '<-';
      T_KEYWORD_OUT = 'puts';
      T_DQUOTE      = '"';
      T_INT         = ('+'|'-')?[0-9]+;
      T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
      T_LITERAL     = [a-zA-Z_][a-zA-Z_0-9]*;
      T_ADD         = '+';
      T_SUB         = '-';
      T_MUL         = '*';
      T_DIV         = '/';
      T_MOD         = '%';
      T_EXP         = '^';
      T_LBR         = '(';
      T_RBR         = ')';
      T_EQ          = '=';
      T_COL         = ',';

      main := |*

        T_KEYWORD_DEF => {
          emit(:T_KEYWORD_DEF)
        };

        T_KEYWORD_END => {
          emit(:T_KEYWORD_END)
        };

        T_KEYWORD_OUT => {
          emit(:T_KEYWORD_OUT)
        };

        T_DQUOTE => {
          emit(:T_DQUOTE)
        };

        T_INT => {
          emit(:T_INT)
        };

        T_FLOAT => {
          emit(:T_FLOAT)
        };

        T_LITERAL => {
          emit(:T_LITERAL)
        };

        T_ADD => {
          emit(:T_ADD)
        };

        T_SUB => {
          emit(:T_SUB)
        };

        T_MUL => {
          emit(:T_MUL)
        };

        T_DIV => {
          emit(:T_DIV)
        };

        T_EXP => {
          emit(:T_EXP)
        };

        T_MOD => {
          emit(:T_MOD)
        };

        T_LBR => {
          emit(:T_LBR)
        };

        T_RBR => {
          emit(:T_RBR)
        };

        T_EQ => {
          emit(:T_EQ)
        };

        T_COL => {
          emit(:T_COL)
        };

        space;

        # any+ => {
          # emit(:UNKNOWN)
        # };

      *|;

    }%%
  end # Lexer
end # CakeLang

%%{

  machine lexer;

  access @;

  T_KEYWORD_DEF = '->';
  T_KEYWORD_END = '<-';
  T_KEYWORD_OUT = 'puts';
  T_DQUOTE      = '"';
  T_INT         = ('+'|'-')?[0-9]+;
  T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
  T_LITERAL     = [a-zA-Z_][a-zA-Z_0-9]*;
  T_ADD         = '+';
  T_SUB         = '-';
  T_MUL         = '*';
  T_DIV         = '/';
  T_MOD         = '%';
  T_EXP         = '^';
  T_LBR         = '(';
  T_RBR         = ')';
  T_EQ          = '=';
  T_COL         = ',';

  main := |*

    T_KEYWORD_DEF => {
      emit(:T_KEYWORD_DEF)
    };

    T_KEYWORD_END => {
      emit(:T_KEYWORD_END)
    };

    T_KEYWORD_OUT => {
      emit(:T_KEYWORD_OUT)
    };

    T_DQUOTE => {
      emit(:T_DQUOTE)
    };

    T_INT => {
      emit(:T_INT)
    };

    T_FLOAT => {
      emit(:T_FLOAT)
    };

    T_LITERAL => {
      emit(:T_LITERAL)
    };

    T_ADD => {
      emit(:T_ADD)
    };

    T_SUB => {
      emit(:T_SUB)
    };

    T_MUL => {
      emit(:T_MUL)
    };

    T_DIV => {
      emit(:T_DIV)
    };

    T_EXP => {
      emit(:T_EXP)
    };

    T_MOD => {
      emit(:T_MOD)
    };

    T_LBR => {
      emit(:T_LBR)
    };

    T_RBR => {
      emit(:T_RBR)
    };

    T_EQ => {
      emit(:T_EQ)
    };

    T_COL => {
      emit(:T_COL)
    };

    space;

    # any+ => {
     # emit(:UNKNOWN)
    # };

  *|;

}%%

module CakeLang
  class Lexer
      CONVERSION = {
          :T_INT => :to_i,
          :T_FLOAT => :to_f
        }

      %% write data;

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      def initialize
        reset
      end

      def reset
        @data     = nil
        @ts       = nil
        @te       = nil
        @tokens   = []
       end

      def emit(symbol)
        # @ts, @te -> token start, token end
        # had to use @te-1 for token range end because 1 digit token (eg.: ',(,),+) and no whitespace between will
        # break the tokenization
        value = @data[@ts..(@te-1)].pack("U*").strip

        if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
          value = value.send(CONVERSION[symbol])
        end

        @tokens << [symbol, value]
      end

      def lex(data)
        @data = data.unpack("c*")
        lexer_start = self.class.lexer_start
        eof = data.length

        %% write init;
        %% write exec;

        tokens = @tokens

        reset

        return tokens
      end
  end
end


Can omit the act variable. Init stack to avoid warnings.

/*
 * Lexes Ragel input files.
 *
 * @LANG: c++
 *
 * Test works with split code gen.
 */

#include <iostream>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

using namespace std;

void escapeXML( const char *data )
{
	while ( *data != 0 ) {
		switch ( *data ) {
			case '<': cout << "<"; break;
			case '>': cout << ">"; break;
			case '&': cout << "&"; break;
			default: cout << *data; break;
		}
		data += 1;
	}
}

void escapeXML( char c )
{
	switch ( c ) {
		case '<': cout << "<"; break;
		case '>': cout << ">"; break;
		case '&': cout << "&"; break;
		default: cout << c; break;
	}
}

void escapeXML( const char *data, int len )
{
	for ( const char *end = data + len; data != end; data++  ) {
		switch ( *data ) {
			case '<': cout << "<"; break;
			case '>': cout << ">"; break;
			case '&': cout << "&"; break;
			default: cout << *data; break;
		}
	}
}

inline void write( const char *data )
{
	cout << data;
}

inline void write( char c )
{
	cout << c;
}

inline void write( const char *data, int len )
{
	cout.write( data, len );
}

%%{
	machine RagelScan;

	word = [a-zA-Z_][a-zA-Z_0-9]*;
	integer = [0-9]+;
	hex = '0x' [0-9a-fA-F] [0-9a-fA-F]*;

	default = ^0;
	EOF = 0;

	# Handles comments in outside code and inline blocks.
	c_comment :=
		( default* :>> '*/' )
		${ escapeXML( fc ); }
		@{ fret; };

	action emit {
		escapeXML( ts, te-ts );
	}

	#
	# Inline action code
	#

	ilscan := |*

		"'" ( [^'\\] | /\\./ )* "'" => emit;
		'"' ( [^"\\] | /\\./ )* '"' => emit;
		'/*' {
			write( "/*" );
			fcall c_comment;
		};
		'//' [^\n]* '\n' => emit;

		'{' {
			write( '{' );
			inline_depth += 1;
		};

		'}' {
			write( '}' );
			/* If dropping down to the last } then return
			 * to ragel code. */
			if ( --inline_depth == 0 ) {
				write( "</inline>\n" );
				fgoto rlscan;
			}
		};

		default => { escapeXML( *ts ); };
	*|;

	#
	# Ragel Tokens
	#

	rlscan := |*
		'}%%' {
			if ( !single_line ) {
				write( "
\n" );
				fgoto main;
			}
		};

		'\n' {
			if ( single_line ) {
				write( "
\n" );
				fgoto main;
			}
		};

		# Word
		word {
			write( "<word>" );
			write( ts, te-ts );
			write( "</word>\n" );
		};

		# Decimal integer.
		integer {
			write( "<int>" );
			write( ts, te-ts );
			write( "</int>\n" );
		};

		# Hexidecimal integer.
		hex {
			write( "<hex>" );
			write( ts, te-ts );
			write( "</hex>\n" );
		};

		# Consume comments.
		'#' [^\n]* '\n';

		# Single literal string.
		"'" ( [^'\\] | /\\./ )* "'" {
			write( "<single_lit>" );
			escapeXML( ts, te-ts );
			write( "</single_lit>\n" );
		};

		# Double literal string.
		'"' ( [^"\\] | /\\./ )* '"' {
			write( "<double_lit>" );
			escapeXML( ts, te-ts );
			write( "</double_lit>\n" );
		};

		# Or literal.
		'[' ( [^\]\\] | /\\./ )* ']' {
			write( "<or_lit>" );
			escapeXML( ts, te-ts );
			write( "</or_lit>\n" );
		};

		# Regex Literal.
		'/' ( [^/\\] | /\\./ ) * '/' {
			write( "<re_lit>" );
			escapeXML( ts, te-ts );
			write( "</re_lit>\n" );
		};

		# Open an inline block
		'{' {
			inline_depth = 1;
			write( "<inline>{" );
			fgoto ilscan;
		};

		punct {
			write( "<symbol>" );
			escapeXML( fc );
			write( "</symbol>\n" );
		};

		default;
	*|;

	#
	# Outside code.
	#

	main := |*

		"'" ( [^'\\] | /\\./ )* "'" => emit;
		'"' ( [^"\\] | /\\./ )* '"' => emit;

		'/*' {
			escapeXML( ts, te-ts );
			fcall c_comment;
		};

		'//' [^\n]* '\n' => emit;

		'%%{' {
			write( "
\n" );
			single_line = false;
			fgoto rlscan;
		};

		'%%' {
			write( "
\n" );
			single_line = true;
			fgoto rlscan;
		};

		default {
			escapeXML( *ts );
		};

		# EOF.
		EOF;
	*|;
}%%

%% write data nofinal;

void test( const char *data )
{
	std::ios::sync_with_stdio(false);

	int cs;
	const char *ts, *te;
	int stack[1], top;
	stack[0] = 0;

	bool single_line = false;
	int inline_depth = 0;

	%% write init;

	/* Read in a block. */
	const char *p = data;
	const char *pe = data + strlen( data );
	const char *eof = pe;
	%% write exec;

	if ( cs == RagelScan_error ) {
		/* Machine failed before finding a token. */
		cerr << "PARSE ERROR" << endl;
		exit(1);
	}
}

#define BUFSIZE 2048

int main()
{
	std::ios::sync_with_stdio(false);

	test("hi %%{ /'}%%'/ { /*{*/ {} } + '\\'' }%%there\n");

	return 0;
}
#ifdef _____OUTPUT_____
hi

<re_lit>/'}%%'/</re_lit>
<inline>{ /*{*/ {} }</inline>
<symbol>+</symbol>
<single_lit>'\''</single_lit>

there
#endif

/*
 * Lexes Ragel input files.
 *
 * @LANG: c++
 *
 * Test works with split code gen.
 */

#include <iostream>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

using namespace std;

void escapeXML( const char *data )
{
	while ( *data != 0 ) {
		switch ( *data ) {
			case '<': cout << "<"; break;
			case '>': cout << ">"; break;
			case '&': cout << "&"; break;
			default: cout << *data; break;
		}
		data += 1;
	}
}

void escapeXML( char c )
{
	switch ( c ) {
		case '<': cout << "<"; break;
		case '>': cout << ">"; break;
		case '&': cout << "&"; break;
		default: cout << c; break;
	}
}

void escapeXML( const char *data, int len )
{
	for ( const char *end = data + len; data != end; data++  ) {
		switch ( *data ) {
			case '<': cout << "<"; break;
			case '>': cout << ">"; break;
			case '&': cout << "&"; break;
			default: cout << *data; break;
		}
	}
}

inline void write( const char *data )
{
	cout << data;
}

inline void write( char c )
{
	cout << c;
}

inline void write( const char *data, int len )
{
	cout.write( data, len );
}

%%{
	machine RagelScan;

	word = [a-zA-Z_][a-zA-Z_0-9]*;
	integer = [0-9]+;
	hex = '0x' [0-9a-fA-F] [0-9a-fA-F]*;

	default = ^0;
	EOF = 0;

	# Handles comments in outside code and inline blocks.
	c_comment :=
		( default* :>> '*/' )
		${ escapeXML( fc ); }
		@{ fret; };

	action emit {
		escapeXML( ts, te-ts );
	}

	#
	# Inline action code
	#

	ilscan := |*

		"'" ( [^'\\] | /\\./ )* "'" => emit;
		'"' ( [^"\\] | /\\./ )* '"' => emit;
		'/*' {
			write( "/*" );
			fcall c_comment;
		};
		'//' [^\n]* '\n' => emit;

		'{' {
			write( '{' );
			inline_depth += 1;
		};

		'}' {
			write( '}' );
			/* If dropping down to the last } then return
			 * to ragel code. */
			if ( --inline_depth == 0 ) {
				write( "</inline>\n" );
				fgoto rlscan;
			}
		};

		default => { escapeXML( *ts ); };
	*|;

	#
	# Ragel Tokens
	#

	rlscan := |*
		'}%%' {
			if ( !single_line ) {
				write( "
\n" );
				fgoto main;
			}
		};

		'\n' {
			if ( single_line ) {
				write( "
\n" );
				fgoto main;
			}
		};

		# Word
		word {
			write( "<word>" );
			write( ts, te-ts );
			write( "</word>\n" );
		};

		# Decimal integer.
		integer {
			write( "<int>" );
			write( ts, te-ts );
			write( "</int>\n" );
		};

		# Hexidecimal integer.
		hex {
			write( "<hex>" );
			write( ts, te-ts );
			write( "</hex>\n" );
		};

		# Consume comments.
		'#' [^\n]* '\n';

		# Single literal string.
		"'" ( [^'\\] | /\\./ )* "'" {
			write( "<single_lit>" );
			escapeXML( ts, te-ts );
			write( "</single_lit>\n" );
		};

		# Double literal string.
		'"' ( [^"\\] | /\\./ )* '"' {
			write( "<double_lit>" );
			escapeXML( ts, te-ts );
			write( "</double_lit>\n" );
		};

		# Or literal.
		'[' ( [^\]\\] | /\\./ )* ']' {
			write( "<or_lit>" );
			escapeXML( ts, te-ts );
			write( "</or_lit>\n" );
		};

		# Regex Literal.
		'/' ( [^/\\] | /\\./ ) * '/' {
			write( "<re_lit>" );
			escapeXML( ts, te-ts );
			write( "</re_lit>\n" );
		};

		# Open an inline block
		'{' {
			inline_depth = 1;
			write( "<inline>{" );
			fgoto ilscan;
		};

		punct {
			write( "<symbol>" );
			escapeXML( fc );
			write( "</symbol>\n" );
		};

		default;
	*|;

	#
	# Outside code.
	#

	main := |*

		"'" ( [^'\\] | /\\./ )* "'" => emit;
		'"' ( [^"\\] | /\\./ )* '"' => emit;

		'/*' {
			escapeXML( ts, te-ts );
			fcall c_comment;
		};

		'//' [^\n]* '\n' => emit;

		'%%{' {
			write( "
\n" );
			single_line = false;
			fgoto rlscan;
		};

		'%%' {
			write( "
\n" );
			single_line = true;
			fgoto rlscan;
		};

		default {
			escapeXML( *ts );
		};

		# EOF.
		EOF;
	*|;
}%%

%% write data nofinal;

void test( const char *data )
{
	std::ios::sync_with_stdio(false);

	int cs, act;
	const char *ts, *te;
	int stack[1], top;

	bool single_line = false;
	int inline_depth = 0;

	%% write init;

	/* Read in a block. */
	const char *p = data;
	const char *pe = data + strlen( data );
	const char *eof = pe;
	%% write exec;

	if ( cs == RagelScan_error ) {
		/* Machine failed before finding a token. */
		cerr << "PARSE ERROR" << endl;
		exit(1);
	}
}

#define BUFSIZE 2048

int main()
{
	std::ios::sync_with_stdio(false);

	test("hi %%{ /'}%%'/ { /*{*/ {} } + '\\'' }%%there\n");

	return 0;
}
#ifdef _____OUTPUT_____
hi

<re_lit>/'}%%'/</re_lit>
<inline>{ /*{*/ {} }</inline>
<symbol>+</symbol>
<single_lit>'\''</single_lit>

there
#endif


eof is not used

eof is not used

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    end
  }

  action consume_null {
    buffer.bytesize < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.bytesize < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer) >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size # how many bytes current message contains in total
        content_length = state.content_length # content length of current message

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = "".force_encoding("BINARY")
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    end
  }

  action consume_null {
    buffer.bytesize < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.bytesize < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer) >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk
        eof = :ignored # end of input

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size # how many bytes current message contains in total
        content_length = state.content_length # content length of current message

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end


Don't save the username until the proper end tag for a username has been seen.

%%{
  machine parser;

  # Leaf tags
  # ---------

  code    = ( ( upper | '_' ) @buffer )+ %saveFailureCode;
  reason  = ( xmlContent @buffer )+ %saveFailureReason;
  pgtIou  = "<cas:proxyGrantingTicket>"
            ( ticketCharacter @buffer ){,256}
            "</cas:proxyGrantingTicket>" %savePgtIou;
  user    = "<cas:user>" ( xmlContent @buffer )+ "</cas:user>" %saveUsername;

  # Non-leaf tags
  # -------------

  serviceResponseStart         = "<cas:serviceResponse xmlns:cas="
                                 quote
                                 "http://www.yale.edu/tp/cas"
                                 quote
                                 ">";
  serviceResponseEnd           = "</cas:serviceResponse>";

  authenticationFailureStart   = "<cas:authenticationFailure code="
                                 quote
                                 code
                                 quote
                                 ">";
  authenticationFailureEnd     = "</cas:authenticationFailure>";

  authenticationSuccessStart   = "<cas:authenticationSuccess>";
  authenticationSuccessEnd     = "</cas:authenticationSuccess>";

  # Top-level elements
  # ------------------

  ok_cas_st     = ( serviceResponseStart
                    space*
                    authenticationSuccessStart
                    space*
                    user
                    space*
                    pgtIou?
                    space*
                    authenticationSuccessEnd
                    space*
                    serviceResponseEnd ) @setAuthenticated;

  failed_cas_st = ( serviceResponseStart
                    space*
                    authenticationFailureStart
                    reason
                    authenticationFailureEnd
                    space*
                    serviceResponseEnd );

  cas_st        = ok_cas_st | failed_cas_st;
}%%

%%{
  machine parser;

  # Leaf tags
  # ---------

  code    = ( ( upper | '_' ) @buffer )+ %saveFailureCode;
  reason  = ( xmlContent @buffer )+ %saveFailureReason;
  pgtIou  = "<cas:proxyGrantingTicket>"
            ( ticketCharacter @buffer ){,256}
            "</cas:proxyGrantingTicket>" %savePgtIou;
  user    = "<cas:user>" ( xmlContent @buffer )+ %saveUsername "</cas:user>";

  # Non-leaf tags
  # -------------

  serviceResponseStart         = "<cas:serviceResponse xmlns:cas="
                                 quote
                                 "http://www.yale.edu/tp/cas"
                                 quote
                                 ">";
  serviceResponseEnd           = "</cas:serviceResponse>";

  authenticationFailureStart   = "<cas:authenticationFailure code="
                                 quote
                                 code
                                 quote
                                 ">";
  authenticationFailureEnd     = "</cas:authenticationFailure>";

  authenticationSuccessStart   = "<cas:authenticationSuccess>";
  authenticationSuccessEnd     = "</cas:authenticationSuccess>";

  # Top-level elements
  # ------------------

  ok_cas_st     = ( serviceResponseStart
                    space*
                    authenticationSuccessStart
                    space*
                    user
                    space*
                    pgtIou?
                    space*
                    authenticationSuccessEnd
                    space*
                    serviceResponseEnd ) @setAuthenticated;

  failed_cas_st = ( serviceResponseStart
                    space*
                    authenticationFailureStart
                    reason
                    authenticationFailureEnd
                    space*
                    serviceResponseEnd );

  cas_st        = ok_cas_st | failed_cas_st;
}%%