Add functionality for parsing simple body

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define MARK_STR_NEW() rb_str_new(mark, MARK_LEN)

#define true 1
#define false 0

typedef struct {
  VALUE error;

  size_t max_message_size;

  const char *chunk;
  const char *p;
  int cs;
  const char *mark;
  const char *mark_key;
  VALUE mark_message;
  size_t mark_message_size;
  size_t mark_message_content_length;
} parser_state_t;

VALUE mStomp = Qnil;
VALUE cMessage = Qnil;
ID g_new;
ID g_write_command;
ID g_write_body;
ID g_build_parse_error;
ID g_max_message_size;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = rb_funcall(cMessage, g_new, 2, Qnil, Qnil);
    mark_message_size = 0;
  }

  action write_command {
    rb_funcall(mark_message, g_write_command, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action mark_key {
    /*
    mk = MARK_STR_NEW;
    mark = NULL;
    */
  }

  action write_header {
    /*
    VALUE value = MARK_STR_NEW;
    mark = NULL;

    rb_funcall(message, rb_intern("write_header"), 2, mk, value);
    mk = Qnil;
    */
  }

  action finish_headers {
    /*
    VALUE length = rb_funcall(message, rb_intern("content_length"), 0);
    if (FIXNUM_P(length)) {
      content_length = FIX2LONG(length);
    } else {
      content_length = -1;
    }
    */
  }

  action write_body {
    rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action consume_null {
    false
    /* ((content_length != -1) && (MARK_LEN < content_length)) */
  }

  action consume_octet {
    true
    /* ((content_length == -1) || (MARK_LEN < content_length)) */
  }

  action check_message_size {
    mark_message_size += 1;
  }

  action finish_message {
    rb_yield(mark_message);
    mark_message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->mark_message);
  rb_gc_mark(state->error);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  VALUE max_message_size;
  rb_scan_args(argc, argv, "01", &max_message_size);

  if (max_message_size == Qnil) {
    max_message_size = rb_funcall(mStomp, g_max_message_size, 0);
  }

  state->error = Qnil;
  state->max_message_size = FIX2LONG(max_message_size);
  state->chunk = NULL;
  state->p = NULL;
  state->cs = start;
  state->mark = NULL;
  state->mark_key = NULL;
  state->mark_message = Qnil;
  state->mark_message_size = 0;
  state->mark_message_content_length = 0;

  return self;
}

static VALUE parser_parse(VALUE self, VALUE chunk) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  if (NIL_P(state->error)) {
    /*
    if state.chunk
      p = state.chunk.bytesize
      chunk = state.chunk << chunk
    else
      p = 0
    end

    pe = chunk.bytesize # special
    */

    // size_t max_message_size = state->max_message_size;
    const char *p = RSTRING_PTR(chunk);
    const char *pe = p + RSTRING_LEN(chunk);

    int cs = state->cs;
    const char *mark = state->mark;
    // mark_key = state->mark_key;
    VALUE mark_message = state->mark_message;
    size_t mark_message_size = state->mark_message_size;
    // mark_content_length = state->mark_content_length;

    %% write exec;

    /*
    if mark
      state.p = chunk.bytesize
      state.chunk = chunk
    else
      state.p = 0
      state.chunk = nil
    end
    */

    state->cs = cs;
    state->mark = mark;
    // state->mark_key = mark_key;
    state->mark_message = mark_message;
    state->mark_message_size = mark_message_size;
    // state->mark_content_length = mark_content_length;

    if (cs == error) {
      long index = p - RSTRING_PTR(chunk);
      state->error = rb_funcall(mStomp, g_build_parse_error, 2, chunk, LONG2NUM(index));
    }
  }

  if ( ! NIL_P(state->error)) {
    rb_exc_raise(state->error);
  }

  return Qnil;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));

  mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));

  g_new = rb_intern("new");
  g_write_command = rb_intern("write_command");
  g_write_body = rb_intern("write_body");
  g_build_parse_error = rb_intern("build_parse_error");
  g_max_message_size = rb_intern("max_message_size");

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "parse", parser_parse, 1);
}

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define MARK_STR_NEW() rb_str_new(mark, MARK_LEN)

#define true 1
#define false 0

typedef struct {
  VALUE error;

  size_t max_message_size;

  const char *chunk;
  const char *p;
  int cs;
  const char *mark;
  const char *mark_key;
  VALUE mark_message;
  size_t mark_message_size;
  size_t mark_message_content_length;
} parser_state_t;

VALUE mStomp = Qnil;
VALUE cMessage = Qnil;
ID g_new;
ID g_write_command;
ID g_build_parse_error;
ID g_max_message_size;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = rb_funcall(cMessage, g_new, 2, Qnil, Qnil);
    mark_message_size = 0;
  }

  action write_command {
    rb_funcall(mark_message, g_write_command, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action mark_key {
    /*
    mk = MARK_STR_NEW;
    mark = NULL;
    */
  }

  action write_header {
    /*
    VALUE value = MARK_STR_NEW;
    mark = NULL;

    rb_funcall(message, rb_intern("write_header"), 2, mk, value);
    mk = Qnil;
    */
  }

  action finish_headers {
    /*
    VALUE length = rb_funcall(message, rb_intern("content_length"), 0);
    if (FIXNUM_P(length)) {
      content_length = FIX2LONG(length);
    } else {
      content_length = -1;
    }
    */
  }

  action write_body {
    /*
    VALUE body = MARK_STR_NEW;
    mark = NULL;
    rb_funcall(message, rb_intern("write_body"), 1, body);
    */
  }

  action consume_null {
    false
    /* ((content_length != -1) && (MARK_LEN < content_length)) */
  }

  action consume_octet {
    true
    /* ((content_length == -1) || (MARK_LEN < content_length)) */
  }

  action check_message_size {
    mark_message_size += 1;
  }

  action finish_message {
    rb_yield(mark_message);
    mark_message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->mark_message);
  rb_gc_mark(state->error);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  VALUE max_message_size;
  rb_scan_args(argc, argv, "01", &max_message_size);

  if (max_message_size == Qnil) {
    max_message_size = rb_funcall(mStomp, g_max_message_size, 0);
  }

  state->error = Qnil;
  state->max_message_size = FIX2LONG(max_message_size);
  state->chunk = NULL;
  state->p = NULL;
  state->cs = start;
  state->mark = NULL;
  state->mark_key = NULL;
  state->mark_message = Qnil;
  state->mark_message_size = 0;
  state->mark_message_content_length = 0;

  return self;
}

static VALUE parser_parse(VALUE self, VALUE chunk) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  if (NIL_P(state->error)) {
    /*
    if state.chunk
      p = state.chunk.bytesize
      chunk = state.chunk << chunk
    else
      p = 0
    end

    pe = chunk.bytesize # special
    */

    // size_t max_message_size = state->max_message_size;
    const char *p = RSTRING_PTR(chunk);
    const char *pe = p + RSTRING_LEN(chunk);

    int cs = state->cs;
    const char *mark = state->mark;
    // mark_key = state->mark_key;
    VALUE mark_message = state->mark_message;
    size_t mark_message_size = state->mark_message_size;
    // mark_content_length = state->mark_content_length;

    %% write exec;

    /*
    if mark
      state.p = chunk.bytesize
      state.chunk = chunk
    else
      state.p = 0
      state.chunk = nil
    end
    */

    state->cs = cs;
    state->mark = mark;
    // state->mark_key = mark_key;
    state->mark_message = mark_message;
    state->mark_message_size = mark_message_size;
    // state->mark_content_length = mark_content_length;

    if (cs == error) {
      long index = p - RSTRING_PTR(chunk);
      state->error = rb_funcall(mStomp, g_build_parse_error, 2, chunk, LONG2NUM(index));
    }
  }

  if ( ! NIL_P(state->error)) {
    rb_exc_raise(state->error);
  }

  return Qnil;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));

  mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));

  g_new = rb_intern("new");
  g_write_command = rb_intern("write_command");
  g_build_parse_error = rb_intern("build_parse_error");
  g_max_message_size = rb_intern("max_message_size");

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "parse", parser_parse, 1);
}


Change ragel to use a scanner instead of a state machine

* This appears to be a better approach since the parsing won't be
  duplicated on the scanner and parser side of things. Ragel is now
  only responsible for tokenizing the input, and Racc will be used
  to parse the tokens.

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'   @{ emit('select') };
    'FROM'     @{ emit('from')   };
    'AS'       @{ emit('as')     };

    # Literals
    'TRUE'     @{ emit('true');  };
    'FALSE'    @{ emit('false'); };
    'NULL'     @{ emit('null');  };

    # Delimiters
    '('        @{ emit('left_paren');  };
    ')'        @{ emit('right_paren'); };
    ','        @{ emit('comma');       };
    '.'        @{ emit('period');      };

    identifier @{ emit('identifier'); };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block

      # Tokenize a stream of input
      while chunk = read_chunk
        optimize_buffer
        @buffer.concat(chunk)
        %% write exec;
      end

      assert_valid_input

      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def emit(token)
      @block.call(token.to_sym, text)
      @ts = nil
    end

    def text
      @buffer[@ts..@p]
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      reset_buffer if @ts.nil?
    end

    def reset_buffer
      @buffer.clear
      @p = 0
    end

    def assert_valid_input
      if @cs < %%{ write first_final; }%%
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL

%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  k_true  = 'TRUE';
  k_false = 'FALSE';
  k_null  = 'NULL';

  action mark               { mark();              }
  action emit_identifier    { emit('identifier');  }
  action emit_select        { emit('select');      }
  action emit_from          { emit('from');        }
  action emit_end           { emit('end');         }
  action call_nested_select { fcall nested_select; }
  action return             { fret;                }

  identifier_part = '"' (alpha (alnum | '""')*) '"';
  identifier      = (identifier_part ('.' identifier_part)*)
                    >mark @emit_identifier;

  alias  = ' ' identifier;
  field  = identifier | k_true | k_false | k_null;
  fields = field (', ' field)*;

  from_clause   = 'FROM' @emit_from ' '
                  (identifier alias? | '(' >call_nested_select alias);

  select_clause = 'SELECT' @emit_select ' ' fields
                  (' ' from_clause)?;

  nested_select := select_clause ')' @emit_end @return;

  main := select_clause;

  # TODO: keep ruby specific variables inline while moving previous code
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable cs   @cs;
  variable top  @top;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = @ts = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass

      @block = block
      stack  = []

      # Tokenize a stream of input
      while chunk = read_chunk
        optimize_buffer
        @buffer.concat(chunk)
        %% write exec;
      end

      assert_valid_input

      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def mark
      @ts = @p
    end

    def emit(token)
      value = @buffer[@ts..@p] if @ts
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
    rescue EOFError
      # return nil
    end

    def optimize_buffer
      if @ts.nil?
        clear_buffer
      elsif @ts.nonzero?
        truncate_buffer
      end
    end

    def clear_buffer
      @buffer.clear
      @p = 0
    end

    def truncate_buffer
      @buffer.slice!(0, @ts)
      @ts = 0
    end

    def assert_valid_input
      if @cs < sql_first_final
        fail InvalidInputError, @input
      end
    end

    # Delegate ragel reader methods to class readers
    (methods(false) | private_methods(false)).each do |method|
      next unless method =~ /\A_?sql_\w+\z/
      define_method(method) { self.class.send(method) }
    end

  end # Scanner
end # SQL


basic error handling

basic error handling

%%{

  machine proto_file;

  action mark {
    mark = p
  }

  action output_mark {
    output(data[mark ... p])
  }

  action outputS_mark {
    outputS(data[mark ... p])
  }

  action dot_to_module {
    output(dot_to_module(data[mark ... p]))
  }

  # WS includes comments too
  WS = ( ("//" (any - "\n")* "\n") | [ \t\r\n] )+ ;

  # only used by strLit
  # TODO: I'm fairly certain these escapes all translate correctly to ruby, but
  # I may be wrong.
  quote = ["'] ;
  hexEscape = "\\" [Xx] [A-Fa-f0-9]{1,2} ;
  octEscape = "\\" "0"? [0-7]{1,3} ;
  charEscape = "\\" [abfnrtv\\\?'"] ;
  strLit = ( quote
            ( hexEscape | octEscape | charEscape | any - [\\\0\n] - quote )*
           quote )
         >mark %output_mark ;

  decInt = ([1-9] digit*) >mark %output_mark ;
  hexInt = ("0" [xX] [A-Fa-f0-9]+) >mark %output_mark ;
  octInt = ("0" [0-7]*) >mark %output_mark ;
  intLit = decInt | hexInt - "0" | octInt ;

  # note that floatLit is a superset of decInt
  floatLit = ( [1-9] digit+ ("." digit+)? ([Ee] [\-+]? digit+)? ) >mark %output_mark ;
  numLit = floatLit | hexInt | octInt ;

  boolLit = ( "true" | "false" ) >mark %output_mark ;

  ident = ( [A-Za-z_] [A-Za-z0-9_]* )
        >{ istart = p } %{ lastIdent = data[istart ... p] }
        @err{ expected["identifier"] };
  constant = boolLit | ident | numLit | strLit ;

  package = "package" WS ident ("." ident)* WS? ";"
          @{ outputS("module #{capfirst(lastIdent)}\n"); @has_package = true };

  reqType = ( "required" | "optional" | "repeated" )
          >mark %outputS_mark %{ output " " }
          @err{ expected["required|optional|repeated"] };

  definedType = ( "double" | "float" | "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" | "bytes" ) >mark %{ output ":" } %output_mark ;
  userType = ( "."? ( ident - definedType ) ( "." ident )* )
           >mark %dot_to_module ;
  fieldType = ( definedType | userType )
            %{ output ", " } ;

  fieldOption = ( "default" %{ output ", :default => " } ) WS? "=" WS? constant ;
  field = reqType WS fieldType WS ( ident %{ output ":#{lastIdent}, " } ) WS? "=" WS? intLit WS? ( "[" WS?  fieldOption WS? "]" )? ";" @{ output "\n" };

  message = "message" WS ident WS? "{" @{ outputS("class #{capfirst(lastIdent)} < ::Protobuf::Message\n"); @depth += 1; outputS("defined_in __FILE__\n"); fgoto messageBody; } ;

  enumField = ( ident %{ outputS(capfirst(lastIdent)) } ) WS? ( "=" %{ output " = " } ) WS? intLit WS? ";" %{ output "\n" } ;

  enum = ( "enum" WS ident WS?
    "{" @{ outputS("class #{capfirst(lastIdent)} < ::Protobuf::Enum\n"); @depth += 1; outputS("defined_in __FILE__\n"); }
    WS? ( ( enumField | ";" ) WS? )*
    "}" ) @{ @depth -= 1; outputS("end\n"); } ;

  messageBody := ( WS? ( ( field | message | enum ) WS? )* "}" )
              @{ @depth -= 1;
                 outputS("end\n");
                 if @depth > 0 then fgoto messageBody ;
                 else fgoto main;
                 end; } ;

  main := WS? ( ( package | message | enum ) WS? )* ;

}%%

class ProtoFileParser
  class ParserError < RuntimeError
  end

  attr_reader :input_name, :io

  def initialize(input_name = nil, io = $stdout)
    @input_name = input_name
    @io = io
  end

  def output(str)
    io.write str
  end

  def dot_to_module(str)
    str.gsub(".", "::")
  end

  def capfirst(str)
    str[0,1].capitalize + str[1..-1]
  end

  def outputS(str)
    io.write "  " if @has_package
    io.write "  " * @depth
    io.write str
  end

  def run_machine(data)
    %%write data;

    %%write init;
    @depth = 0
    @has_package = false
    istart = 0
    mark = 0
    eof = -1

    expected = proc do |desc|
      lines = data.split("\n")
      lp = p
      line = 0
      (lp -= lines[line].length + 1; line += 1) while lines[line].length < lp
      if desc
        str = "\n#{line}:error: expected #{desc}\n#{lines[line]}\n"
      else
        str = "\n#{line}:error: parse error\n#{lines[line]}\n"
      end
      str += " " * lp
      str += "^\n"
      raise ParserError, str
    end

    io.puts "# Generated file. Do not edit."
    if input_name
      io.puts "# <#{input_name}>"
    end
    io.puts
    io.puts "require 'protobuf/message/message'"
    io.puts "require 'protobuf/message/enum'"
    io.puts "require 'protobuf/message/service'"
    io.puts "require 'protobuf/message/extend'"
    io.puts

    %%write exec;

    if cs < %%write first_final;
      expected[nil]
    end

    output "end\n" if @has_package
  end

end

begin
  ProtoFileParser.new(ARGV[0]).run_machine(File.read(ARGV[0]))
rescue ProtoFileParser::ParserError
  puts $!.message
end
# ProtoFileParser.new.run_machine(File.read("/Users/bpalmer/Programming/ctriton/protobufs/manifest.proto"))

%%{

  machine proto_file;

  action mark {
    mark = p
  }

  action output_mark {
    output(data[mark ... p])
  }

  action outputS_mark {
    outputS(data[mark ... p])
  }

  action dot_to_module {
    output(dot_to_module(data[mark ... p]))
  }

  # WS includes comments too
  WS = ( ("//" (any - "\n")* "\n") | [ \t\r\n] )+ ;

  # only used by strLit
  # TODO: I'm fairly certain these escapes all translate correctly to ruby, but
  # I may be wrong.
  quote = ["'] ;
  hexEscape = "\\" [Xx] [A-Fa-f0-9]{1,2} ;
  octEscape = "\\" "0"? [0-7]{1,3} ;
  charEscape = "\\" [abfnrtv\\\?'"] ;
  strLit = ( quote
            ( hexEscape | octEscape | charEscape | any - [\\\0\n] - quote )*
           quote )
         >mark %output_mark ;

  decInt = ([1-9] digit*) >mark %output_mark ;
  hexInt = ("0" [xX] [A-Fa-f0-9]+) >mark %output_mark ;
  octInt = ("0" [0-7]*) >mark %output_mark ;
  intLit = decInt | hexInt - "0" | octInt ;

  # note that floatLit is a superset of decInt
  floatLit = ( [1-9] digit+ ("." digit+)? ([Ee] [\-+]? digit+)? ) >mark %output_mark ;
  numLit = floatLit | hexInt | octInt ;

  boolLit = ( "true" | "false" ) >mark %output_mark ;

  ident = ( [A-Za-z_] [A-Za-z0-9_]* )
        >{ istart = p } %{ lastIdent = data[istart ... p] } ;
  constant = boolLit | ident | numLit | strLit ;

  package = "package" WS ident ("." ident)* WS? ";"
          @{ outputS("module #{capfirst(lastIdent)}\n"); @has_package = true };

  reqType = ( "required" | "optional" | "repeated" )
          >mark %outputS_mark %{ output " " };

  definedType = ( "double" | "float" | "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" | "bytes" ) >mark %{ output ":" } %output_mark ;
  userType = ( "."? ( ident - definedType ) ( "." ident )* )
           >mark %dot_to_module ;
  fieldType = ( definedType | userType )
            %{ output ", " } ;

  fieldOption = ( "default" %{ output ", :default => " } ) WS? "=" WS? constant ;
  field = reqType WS fieldType WS ( ident %{ output ":#{lastIdent}, " } ) WS? "=" WS? intLit WS? ( "[" WS?  fieldOption WS? "]" )? ";" @{ output "\n" };

  message = "message" WS ident WS? "{" @{ outputS("class #{capfirst(lastIdent)} < ::Protobuf::Message\n"); @depth += 1; outputS("defined_in __FILE__\n"); fgoto messageBody; } ;

  enumField = ( ident %{ outputS(capfirst(lastIdent)) } ) WS? ( "=" %{ output " = " } ) WS? intLit WS? ";" %{ output "\n" } ;

  enum = ( "enum" WS ident WS?
    "{" @{ outputS("class #{capfirst(lastIdent)} < ::Protobuf::Enum\n"); @depth += 1; outputS("defined_in __FILE__\n"); }
    ( WS? ( enumField | ";" ) WS? )*
    "}" ) @{ @depth -= 1; outputS("end\n"); } ;

  messageBody := ( ( WS? ( field | message | enum ) WS? )* "}" )
              @{ @depth -= 1;
                 outputS("end\n");
                 if @depth > 0 then fgoto messageBody ;
                 else fgoto main;
                 end; } ;

  main := WS? ( ( package | message | enum ) WS? )* ;

}%%

class ProtoFileParser
  attr_reader :input_name, :io

  def initialize(input_name = nil, io = $stdout)
    @input_name = input_name
    @io = io
  end

  def output(str)
    io.write str
  end

  def dot_to_module(str)
    str.gsub(".", "::")
  end

  def capfirst(str)
    str[0,1].capitalize + str[1..-1]
  end

  def outputS(str)
    io.write "  " if @has_package
    io.write "  " * @depth
    io.write str
  end

  def run_machine(data)
    %%write data;

    %%write init;
    @depth = 0
    @has_package = false
    istart = 0
    mark = 0
    eof = -1

    io.puts "# Generated file. Do not edit."
    if input_name
      io.puts "# <#{input_name}>"
    end
    io.puts
    io.puts "require 'protobuf/message/message'"
    io.puts "require 'protobuf/message/enum'"
    io.puts "require 'protobuf/message/service'"
    io.puts "require 'protobuf/message/extend'"
    io.puts

    %%write exec;

    if cs < %%{ write first_final; }%%
      raise "Could not parse proto file. Syntax error? Or BUG???"
    end

    output "end\n" if @has_package
  end

end

ProtoFileParser.new("__DATA__").run_machine(DATA.read)
# ProtoFileParser.new.run_machine(File.read("/Users/bpalmer/Programming/ctriton/protobufs/manifest.proto"))
__END__


Change ragel version of csv_project3

Change ragel version of csv_project3

#include "../common.h"

#define P(C) fputs(C, stdout);

int echo = 0;

%%{
  machine csv_project3;

  action echo_on  { echo = 1; }
  action echo_off { echo = 0; }
  action dump {
    if(echo) {
      // fprintf(stdout, "%.*s", 1, &fc);
      fprintf(stdout, "%c", fc);
    }
  }

  col      = [^,\n]*;
  dropCol  = col > echo_off;
  keepCol  = col > echo_on % echo_off;
  row      = dropCol ','
             keepCol ',' > { P("\t") }
             dropCol ','
             dropCol ','
             keepCol ','
             dropCol;
  csvProj3 = (row '\n' > { P("\n") })*;

  main := csvProj3 $ dump;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;

    if(p != pe) {
      FAIL;
    }
  }

  POST;

  return 0;
}

#include "../common.h"

#define P(C) fputs(C, stdout);

int echo = 0;

%%{
  machine csv_project3;

  action echo_on  { echo = 1; }
  action echo_off { echo = 0; }
  action dump {
    if(echo) {
      fprintf(stdout, "%.*s", 1, &fc);
    }
  }

  col      = [^,\n]*;
  dropCol  = col > echo_off;
  keepCol  = col > echo_on % echo_off;
  row      = dropCol ','
             keepCol ',' > { P("\t") }
             dropCol ','
             dropCol ','
             keepCol ','
             dropCol;
  csvProj3 = (row '\n' > { P("\n") })*;

  main := csvProj3 $ dump;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;

    if(p != pe) {
      FAIL;
    }
  }

  POST;

  return 0;
}


Re-wrapped comments in the XML lexer.

Re-wrapped comments in the XML lexer.

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    class Lexer
      %% write data; # %

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Array]
      #
      HTML_VOID_ELEMENTS = [
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ]

      # Lazy way of forwarding instance method calls used internally by Ragel
      # to their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      ##
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @data     = nil
        @ts       = nil
        @te       = nil
        @tokens   = []
        @stack    = []
        @top      = 0
        @elements = []

        @buffer_start_position = nil
      end

      ##
      # Lexes the supplied String and returns an Array of tokens. Each token is
      # an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # @param [String] data The string to lex.
      # @return [Array]
      #
      def lex(data)
        @data       = data.unpack('U*')
        lexer_start = self.class.lexer_start
        eof         = data.length

        %% write init;
        %% write exec;

        tokens = @tokens

        reset

        return tokens
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start = @ts, stop = @te)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # By default `@ts` and `@te` are used as the start/stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start = @ts, stop = @te)
        return @data[start...stop].pack('U*')
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @tokens << token
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer, set to `@te`
      #  by default.
      #
      def start_buffer(position = @te)
        @buffer_start_position = position
      end

      ##
      # Returns `true` if we're currently buffering.
      #
      # @return [TrueClass|FalseClass]
      #
      def buffering?
        return !!@buffer_start_position
      end

      ##
      # Emits the current buffer if we have any. The current line number is
      # advanced based on the amount of newlines in the buffer.
      #
      # @param [Fixnum] position The end position of the buffer, set to `@ts`
      #  by default.
      #
      # @param [Symbol] type The type of node to emit.
      #
      def emit_buffer(position = @ts, type = :T_TEXT)
        return unless @buffer_start_position

        content = text(@buffer_start_position, position)

        unless content.empty?
          add_token(type, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end

        @buffer_start_position = nil
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        # Use instance variables for `ts` and friends.
        access @;
        getkey (@data[p] || 0);

        newline    = '\n' | '\r\n';
        whitespace = [ \t];

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        action start_string_dquote {
          start_buffer

          fcall string_dquote;
        }

        action start_string_squote {
          start_buffer

          fcall string_squote;
        }

        # Machine for processing double quoted strings.
        string_dquote := |*
          dquote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          squote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

        action start_doctype {
          emit_buffer
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE) };

          # Lex the public/system IDs as regular strings.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          emit_buffer
          add_token(:T_CDATA_START)

          start_buffer

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          cdata_end => {
            emit_buffer
            add_token(:T_CDATA_END)

            fret;
          };

          any;
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          emit_buffer
          add_token(:T_COMMENT_START)

          start_buffer

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          comment_end => {
            emit_buffer
            add_token(:T_COMMENT_END)

            fret;
          };

          any;
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          emit_buffer
          add_token(:T_XML_DECL_START)

          start_buffer

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            emit_buffer
            add_token(:T_XML_DECL_END)

            fret;
          };

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          emit_buffer
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(@ts + 1)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_name  = [a-zA-Z0-9\-_:]+;
        element_start = '<' element_name;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          element_name => { emit(:T_ATTR) };

          # Attribute values.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' element_name '>' => {
            emit_buffer
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          any => {
            # First character, start buffering (unless we already are buffering).
            start_buffer(@ts) unless buffering?

            # EOF, emit the text buffer.
            if @te == eof
              emit_buffer(@te)
            end
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option). To
    # lex HTML input set the `:html` option to `true` when creating an instance
    # of the lexer:
    #
    #     lexer = Oga::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    class Lexer
      %% write data; # %

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Array]
      #
      HTML_VOID_ELEMENTS = [
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ]

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      ##
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      #  the input as HTML instead of SGML/XML. This makes it possible to lex
      #  HTML void elements such as `<link href="">`.
      #
      def initialize(options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to call
      # this method yourself as its called by #lex after lexing a given String.
      #
      def reset
        @line     = 1
        @data     = nil
        @ts       = nil
        @te       = nil
        @tokens   = []
        @stack    = []
        @top      = 0
        @elements = []

        @buffer_start_position = nil
      end

      ##
      # Lexes the supplied String and returns an Array of tokens. Each token is
      # an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # @param [String] data The string to lex.
      # @return [Array]
      #
      def lex(data)
        @data       = data.unpack('U*')
        lexer_start = self.class.lexer_start
        eof         = data.length

        %% write init;
        %% write exec;

        tokens = @tokens

        reset

        return tokens
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start = @ts, stop = @te)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # By default `@ts` and `@te` are used as the start/stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start = @ts, stop = @te)
        return @data[start...stop].pack('U*')
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @tokens << token
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer, set to `@te`
      #  by default.
      #
      def start_buffer(position = @te)
        @buffer_start_position = position
      end

      ##
      # Returns `true` if we're currently buffering.
      #
      # @return [TrueClass|FalseClass]
      #
      def buffering?
        return !!@buffer_start_position
      end

      ##
      # Emits the current buffer if we have any. The current line number is
      # advanced based on the amount of newlines in the buffer.
      #
      # @param [Fixnum] position The end position of the buffer, set to `@ts` by
      #  default.
      #
      # @param [Symbol] type The type of node to emit.
      #
      def emit_buffer(position = @ts, type = :T_TEXT)
        return unless @buffer_start_position

        content = text(@buffer_start_position, position)

        unless content.empty?
          add_token(type, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end

        @buffer_start_position = nil
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        # Use instance variables for `ts` and friends.
        access @;
        getkey (@data[p] || 0);

        newline    = '\n' | '\r\n';
        whitespace = [ \t];

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type of
        # quote.
        dquote = '"';
        squote = "'";

        action start_string_dquote {
          start_buffer

          fcall string_dquote;
        }

        action start_string_squote {
          start_buffer

          fcall string_squote;
        }

        # Machine for processing double quoted strings.
        string_dquote := |*
          dquote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          squote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

        action start_doctype {
          emit_buffer
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public and
        # system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE) };

          # Lex the public/system IDs as regular strings.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          emit_buffer
          add_token(:T_CDATA_START)

          start_buffer

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          cdata_end => {
            emit_buffer
            add_token(:T_CDATA_END)

            fret;
          };

          any;
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the end
        # tag.
        #
        # Unlike the W3 specification these rules *do* allow character sequences
        # such as `--` and `->`. Putting extra checks in for these sequences
        # would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          emit_buffer
          add_token(:T_COMMENT_START)

          start_buffer

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          comment_end => {
            emit_buffer
            add_token(:T_COMMENT_END)

            fret;
          };

          any;
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          emit_buffer
          add_token(:T_XML_DECL_START)

          start_buffer

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            emit_buffer
            add_token(:T_XML_DECL_END)

            fret;
          };

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and namespace
        # (if any). Remaining work is delegated to a dedicated machine.
        action start_element {
          emit_buffer
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(@ts + 1)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_name  = [a-zA-Z0-9\-_:]+;
        element_start = '<' element_name;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          element_name => { emit(:T_ATTR) };

          # Attribute values.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' element_name '>' => {
            emit_buffer
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Note that this rule should be declared at the very bottom as it will
          # otherwise take precedence over the other rules.
          any => {
            # First character, start buffering (unless we already are buffering).
            start_buffer(@ts) unless buffering?

            # EOF, emit the text buffer.
            if @te == eof
              emit_buffer(@te)
            end
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga


dynlease: Add a missing std::move()

dynlease: Add a missing std::move()

#include <unistd.h>
#include <time.h>
#include <cstdio>
#include <string>
#include <vector>
#include <unordered_map>
#include <algorithm>
#include <cassert>
#include <nk/scopeguard.hpp>
#include <nk/from_string.hpp>
#include <nk/net/ip_address.hpp>
extern "C" {
#include "nk/log.h"
}

#define MAX_LINE 2048

extern int64_t get_current_ts();

struct lease_state_v4
{
    lease_state_v4(nk::ip_address &&addr_, const std::string &ma, int64_t et)
        : addr(std::move(addr_)), expire_time(et)
    {
        assert(ma.size() == 6);
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(nk::ip_address &&addr_, const uint8_t *ma, int64_t et)
        : addr(std::move(addr_)), expire_time(et)
    {
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(const nk::ip_address &addr_, const std::string &ma, int64_t et)
        : addr(addr_), expire_time(et)
    {
        assert(ma.size() == 6);
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(const nk::ip_address &addr_, const uint8_t *ma, int64_t et)
        : addr(addr_), expire_time(et)
    {
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    nk::ip_address addr;
    uint8_t macaddr[6];
    int64_t expire_time;
};

struct lease_state_v6
{
    lease_state_v6(nk::ip_address &&addr_, std::string &&duid_, uint32_t iaid_, int64_t et)
        : addr(std::move(addr_)), duid(std::move(duid_)), iaid(iaid_), expire_time(et) {}
    lease_state_v6(const nk::ip_address &addr_, const std::string &duid_, uint32_t iaid_, int64_t et)
        : addr(addr_), duid(duid_), iaid(iaid_), expire_time(et) {}
    nk::ip_address addr;
    std::string duid;
    uint32_t iaid;
    int64_t expire_time;
};

// These vectors are sorted by addr.
using dynlease_map_v4 = std::vector<lease_state_v4>;
using dynlease_map_v6 = std::vector<lease_state_v6>;

// Maps interfaces to lease data.
static std::unordered_map<std::string, dynlease_map_v4> dyn_leases_v4;
static std::unordered_map<std::string, dynlease_map_v6> dyn_leases_v6;

static bool emplace_dynlease_state(size_t linenum, std::string &&interface,
                                   std::string &&v4_addr, const std::string &macaddr,
                                   int64_t expire_time)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) {
        auto x = dyn_leases_v4.emplace(std::make_pair(std::move(interface), dynlease_map_v4()));
        si = x.first;
    }
    nk::ip_address ipa;
    if (!ipa.from_string(v4_addr)) {
        log_line("Bad IP address at line %zu: %s", linenum, v4_addr.c_str());
        return false;
    }
    // We won't get duplicates unless someone manually edits the file.  If they do,
    // then they get what they deserve.
    si->second.emplace_back(std::move(ipa), macaddr, expire_time);
    return true;
}

static bool emplace_dynlease_state(size_t linenum, std::string &&interface,
                                   std::string &&v6_addr, std::string &&duid,
                                   uint32_t iaid, int64_t expire_time)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) {
        auto x = dyn_leases_v6.emplace(std::make_pair(std::move(interface), dynlease_map_v6()));
        si = x.first;
    }
    nk::ip_address ipa;
    if (!ipa.from_string(v6_addr)) {
        log_line("Bad IP address at line %zu: %s", linenum, v6_addr.c_str());
        return false;
    }
    si->second.emplace_back(std::move(ipa), std::move(duid), iaid, expire_time);
    return true;
}

size_t dynlease4_count(const std::string &interface)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end())
        return 0;
    return si->second.size();
}

size_t dynlease6_count(const std::string &interface)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end())
        return 0;
    return si->second.size();
}

bool dynlease_add(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr,
                  int64_t expire_time)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) {
        auto x = dyn_leases_v4.emplace(std::make_pair(interface, dynlease_map_v4()));
        si = x.first;
    }

    for (auto &i: si->second) {
        if (i.addr == v4_addr) {
            if (memcmp(&i.macaddr, macaddr, 6) == 0) {
                i.expire_time = expire_time;
                return true;
            }
            return false;
        }
    }
    si->second.emplace_back(std::move(v4_addr), macaddr, expire_time);
    return true;
}

bool dynlease_add(const std::string &interface, const nk::ip_address &v6_addr,
                  const std::string &duid, uint32_t iaid, int64_t expire_time)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) {
        auto x = dyn_leases_v6.emplace(std::make_pair(std::move(interface), dynlease_map_v6()));
        si = x.first;
    }

    for (auto &i: si->second) {
        if (i.addr == v6_addr) {
            if (i.duid == duid && i.iaid == iaid) {
                i.expire_time = expire_time;
                return true;
            }
            return false;
        }
    }
    si->second.emplace_back(std::move(v6_addr), duid, iaid, expire_time);
    return true;
}

const nk::ip_address &dynlease_query_refresh(const std::string &interface, const uint8_t *macaddr,
                                   int64_t expire_time)
{
    static nk::ip_address blank{};
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return blank;

    for (auto &i: si->second) {
        if (memcmp(&i.macaddr, macaddr, 6) == 0) {
            i.expire_time = expire_time;
            return i.addr;
        }
        return blank;
    }
    return blank;
}

const nk::ip_address &dynlease_query_refresh(const std::string &interface, const std::string &duid,
                                   uint32_t iaid, int64_t expire_time)
{
    static nk::ip_address blank{};
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return blank;

    for (auto &i: si->second) {
        if (i.duid == duid && i.iaid == iaid) {
            i.expire_time = expire_time;
            return i.addr;
        }
        return blank;
    }
    return blank;
}

bool dynlease_exists(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return false;

    for (auto &i: si->second) {
        if (i.addr == v4_addr && memcmp(&i.macaddr, macaddr, 6) == 0) {
            return get_current_ts() < i.expire_time;
        }
    }
    return false;
}

bool dynlease_exists(const std::string &interface, const nk::ip_address &v6_addr,
                     const std::string &duid, uint32_t iaid)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return false;

    for (auto &i: si->second) {
        if (i.addr == v6_addr && i.duid == duid && i.iaid == iaid) {
            return get_current_ts() < i.expire_time;
        }
    }
    return false;
}

bool dynlease_del(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return false;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == v4_addr && memcmp(&i->macaddr, macaddr, 6) == 0) {
            si->second.erase(i);
            return true;
        }
    }
    return false;
}

bool dynlease_del(const std::string &interface, const nk::ip_address &v6_addr,
                  const std::string &duid, uint32_t iaid)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return false;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == v6_addr && i->duid == duid && i->iaid == iaid) {
            si->second.erase(i);
            return true;
        }
    }
    return false;
}

bool dynlease_unused_addr(const std::string &interface, const nk::ip_address &addr)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return true;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == addr)
            return false;
    }
    return true;
}

// v4 <interface> <ip> <macaddr> <expire_time>
// v6 <interface> <ip> <duid> <iaid> <expire_time>

bool dynlease_serialize(const std::string &path)
{
    const auto tmp_path = path + ".tmp";
    const auto f = fopen(tmp_path.c_str(), "w");
    if (!f) {
        log_line("%s: failed to open '%s' for dynamic lease serialization\n",
                 __func__, path.c_str());
        return false;
    }
    SCOPE_EXIT{ fclose(f); unlink(tmp_path.c_str()); };
    for (const auto &i: dyn_leases_v4) {
        const auto &iface = i.first;
        const auto &ls = i.second;
        for (const auto &j: ls) {
            // Don't write out dynamic leases that have expired.
            if (get_current_ts() >= j.expire_time)
                continue;
            char wbuf[1024];
            int splen = snprintf(wbuf, sizeof wbuf, "v4 %s %s %2.x%2.x%2.x%2.x%2.x%2.x %zu\n",
                                 iface.c_str(), j.addr.to_string().c_str(),
                                 j.macaddr[0], j.macaddr[1], j.macaddr[2],
                                 j.macaddr[3], j.macaddr[4], j.macaddr[5], j.expire_time);
            if (splen < 0)
                suicide("%s: snprintf failed; return=%d", __func__, splen);
            if ((size_t)splen >= sizeof wbuf)
                suicide("%s: snprintf dest buffer too small %d >= %zu",
                        __func__, splen, sizeof wbuf);
            const auto fs = fwrite(wbuf, 1, splen, f);
            if (fs != (size_t)splen) {
                log_line("%s: short write %zd < %zu\n", __func__, fs, sizeof wbuf);
                return false;
            }
        }
    }
    for (const auto &i: dyn_leases_v6) {
        const auto &iface = i.first;
        const auto &ls = i.second;
        for (const auto &j: ls) {
            // Don't write out dynamic leases that have expired.
            if (get_current_ts() >= j.expire_time)
                continue;

            std::string wbuf;
            wbuf.append("v6 ");
            wbuf.append(iface);
            wbuf.append(" ");
            wbuf.append(j.addr.to_string());
            wbuf.append(" ");
            for (const auto &k: j.duid) {
                char tbuf[16];
                snprintf(tbuf, sizeof tbuf, "%.2hhx", k);
                wbuf.append(tbuf);
            }
            wbuf.append(" ");
            wbuf.append(std::to_string(j.iaid));
            wbuf.append(" ");
            wbuf.append(std::to_string(j.expire_time));
            wbuf.append("\n");
            const auto fs = fwrite(wbuf.c_str(), 1, wbuf.size(), f);
            if (fs != wbuf.size()) {
                log_line("%s: short write %zd < %zu", __func__, fs, wbuf.size());
                return false;
            }
        }
    }
    if (fflush(f)) {
        log_line("%s: fflush failed: %s", __func__, strerror(errno));
        return false;
    }
    const auto fd = fileno(f);
    if (fdatasync(fd)) {
        log_line("%s: fdatasync failed: %s", __func__, strerror(errno));
        return false;
    }
    if (rename(tmp_path.c_str(), path.c_str())) {
        log_line("%s: rename failed: %s", __func__, strerror(errno));
        return false;
    }
    return true;
}

// v4 <interface> <ip> <macaddr> <expire_time>
// v6 <interface> <ip> <duid> <iaid> <expire_time>

struct cfg_parse_state {
    cfg_parse_state() : st(nullptr), cs(0), parse_error(false) {}
    void newline() {
        duid.clear();
        macaddr.clear();
        v4_addr.clear();
        v6_addr.clear();
        interface.clear();
        iaid = 0;
        expire_time = 0;
        parse_error = false;
    }
    const char *st;
    int cs;

    std::string duid;
    std::string macaddr;
    std::string v4_addr;
    std::string v6_addr;
    std::string interface;
    int64_t expire_time;
    uint32_t iaid;
    bool parse_error;
};

static inline std::string lc_string(const char *s, size_t slen)
{
    auto r = std::string(s, slen);
    for (auto &i: r) i = tolower(i);
    return r;
}

%%{
    machine dynlease_line_m;
    access cps.;

    action St { cps.st = p; }

    action InterfaceEn { cps.interface = std::string(cps.st, p - cps.st); }
    action DuidEn { cps.duid = lc_string(cps.st, p - cps.st); }
    action IaidEn {
        if (auto t = nk::from_string<uint32_t>(lc_string(cps.st, p - cps.st))) cps.iaid = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }
    action MacAddrEn { cps.macaddr = lc_string(cps.st, p - cps.st); }
    action V4AddrEn { cps.v4_addr = lc_string(cps.st, p - cps.st); }
    action V6AddrEn { cps.v6_addr = lc_string(cps.st, p - cps.st); }
    action ExpireTimeEn {
        if (auto t = nk::from_string<int64_t>(std::string(cps.st, p - cps.st))) cps.expire_time = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }

    action V4EntryEn {
        emplace_dynlease_state(linenum, std::move(cps.interface), std::move(cps.v4_addr),
                               std::move(cps.macaddr), cps.expire_time);
    }
    action V6EntryEn {
        emplace_dynlease_state(linenum, std::move(cps.interface), std::move(cps.v6_addr),
                               std::move(cps.duid), cps.iaid, cps.expire_time);
    }

    interface = alnum+ >St %InterfaceEn;
    duid = (xdigit+ | (xdigit{2} ('-' xdigit{2})*)+) >St %DuidEn;
    iaid = digit+ >St %IaidEn;
    macaddr = ((xdigit{2} ':'){5} xdigit{2}) >St %MacAddrEn;
    v4_addr = (digit{1,3} | '.')+ >St %V4AddrEn;
    v6_addr = (xdigit{1,4} | ':')+ >St %V6AddrEn;
    expire_time = digit+ >St %ExpireTimeEn;

    v4_entry = space* 'v4' space+ interface space+ v4_addr space+ macaddr space+ expire_time space*;
    v6_entry = space* 'v6' space+ interface space+ v6_addr space+ duid space+ iaid space+ expire_time space*;

    main := v4_entry %V4EntryEn | v6_entry %V6EntryEn;
}%%

%% write data;

static int do_parse_dynlease_line(cfg_parse_state &cps, const char *p, size_t plen,
                             const size_t linenum)
{
    const char *pe = p + plen;
    const char *eof = pe;

    %% write init;
    %% write exec;

    if (cps.parse_error) return -1;
    if (cps.cs >= dynlease_line_m_first_final)
        return 1;
    if (cps.cs == dynlease_line_m_error)
        return -1;
    return -2;
}

bool dynlease_deserialize(const std::string &path)
{
    char buf[MAX_LINE];
    const auto f = fopen(path.c_str(), "r");
    if (!f) {
        log_line("%s: failed to open '%s' for dynamic lease deserialization\n",
                 __func__, path.c_str());
        return false;
    }
    SCOPE_EXIT{ fclose(f); };
    dyn_leases_v4.clear();
    dyn_leases_v6.clear();
    size_t linenum = 0;
    cfg_parse_state ps;
    while (!feof(f)) {
        if (!fgets(buf, sizeof buf, f)) {
            if (!feof(f))
                log_line("%s: io error fetching line of '%s'\n", __func__, path.c_str());
            break;
        }
        auto llen = strlen(buf);
        if (llen == 0)
            continue;
        if (buf[llen-1] == '\n')
            buf[--llen] = 0;
        ++linenum;
        ps.newline();
        const auto r = do_parse_dynlease_line(ps, buf, llen, linenum);
        if (r < 0) {
            if (r == -2)
                log_line("%s: Incomplete dynlease at line %zu; ignoring",
                         __func__, linenum);
            else
                log_line("%s: Malformed dynlease at line %zu; ignoring.",
                         __func__, linenum);
            continue;
        }
    }
    return true;
}

#include <unistd.h>
#include <time.h>
#include <cstdio>
#include <string>
#include <vector>
#include <unordered_map>
#include <algorithm>
#include <cassert>
#include <nk/scopeguard.hpp>
#include <nk/from_string.hpp>
#include <nk/net/ip_address.hpp>
extern "C" {
#include "nk/log.h"
}

#define MAX_LINE 2048

extern int64_t get_current_ts();

struct lease_state_v4
{
    lease_state_v4(nk::ip_address &&addr_, const std::string &ma, int64_t et)
        : addr(std::move(addr_)), expire_time(et)
    {
        assert(ma.size() == 6);
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(nk::ip_address &&addr_, const uint8_t *ma, int64_t et)
        : addr(std::move(addr_)), expire_time(et)
    {
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(const nk::ip_address &addr_, const std::string &ma, int64_t et)
        : addr(addr_), expire_time(et)
    {
        assert(ma.size() == 6);
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    lease_state_v4(const nk::ip_address &addr_, const uint8_t *ma, int64_t et)
        : addr(addr_), expire_time(et)
    {
        for (unsigned i = 0; i < 6; ++i)
            macaddr[i] = ma[i];
    }
    nk::ip_address addr;
    uint8_t macaddr[6];
    int64_t expire_time;
};

struct lease_state_v6
{
    lease_state_v6(nk::ip_address &&addr_, std::string &&duid_, uint32_t iaid_, int64_t et)
        : addr(std::move(addr_)), duid(std::move(duid_)), iaid(iaid_), expire_time(et) {}
    lease_state_v6(const nk::ip_address &addr_, const std::string &duid_, uint32_t iaid_, int64_t et)
        : addr(addr_), duid(duid_), iaid(iaid_), expire_time(et) {}
    nk::ip_address addr;
    std::string duid;
    uint32_t iaid;
    int64_t expire_time;
};

// These vectors are sorted by addr.
using dynlease_map_v4 = std::vector<lease_state_v4>;
using dynlease_map_v6 = std::vector<lease_state_v6>;

// Maps interfaces to lease data.
static std::unordered_map<std::string, dynlease_map_v4> dyn_leases_v4;
static std::unordered_map<std::string, dynlease_map_v6> dyn_leases_v6;

static bool emplace_dynlease_state(size_t linenum, std::string &&interface,
                                   std::string &&v4_addr, const std::string &macaddr,
                                   int64_t expire_time)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) {
        auto x = dyn_leases_v4.emplace(std::make_pair(std::move(interface), dynlease_map_v4()));
        si = x.first;
    }
    nk::ip_address ipa;
    if (!ipa.from_string(v4_addr)) {
        log_line("Bad IP address at line %zu: %s", linenum, v4_addr.c_str());
        return false;
    }
    // We won't get duplicates unless someone manually edits the file.  If they do,
    // then they get what they deserve.
    si->second.emplace_back(std::move(ipa), macaddr, expire_time);
    return true;
}

static bool emplace_dynlease_state(size_t linenum, std::string &&interface,
                                   std::string &&v6_addr, std::string &&duid,
                                   uint32_t iaid, int64_t expire_time)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) {
        auto x = dyn_leases_v6.emplace(std::make_pair(std::move(interface), dynlease_map_v6()));
        si = x.first;
    }
    nk::ip_address ipa;
    if (!ipa.from_string(v6_addr)) {
        log_line("Bad IP address at line %zu: %s", linenum, v6_addr.c_str());
        return false;
    }
    si->second.emplace_back(std::move(ipa), std::move(duid), iaid, expire_time);
    return true;
}

size_t dynlease4_count(const std::string &interface)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end())
        return 0;
    return si->second.size();
}

size_t dynlease6_count(const std::string &interface)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end())
        return 0;
    return si->second.size();
}

bool dynlease_add(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr,
                  int64_t expire_time)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) {
        auto x = dyn_leases_v4.emplace(std::make_pair(interface, dynlease_map_v4()));
        si = x.first;
    }

    for (auto &i: si->second) {
        if (i.addr == v4_addr) {
            if (memcmp(&i.macaddr, macaddr, 6) == 0) {
                i.expire_time = expire_time;
                return true;
            }
            return false;
        }
    }
    si->second.emplace_back(std::move(v4_addr), macaddr, expire_time);
    return true;
}

bool dynlease_add(const std::string &interface, const nk::ip_address &v6_addr,
                  const std::string &duid, uint32_t iaid, int64_t expire_time)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) {
        auto x = dyn_leases_v6.emplace(std::make_pair(std::move(interface), dynlease_map_v6()));
        si = x.first;
    }

    for (auto &i: si->second) {
        if (i.addr == v6_addr) {
            if (i.duid == duid && i.iaid == iaid) {
                i.expire_time = expire_time;
                return true;
            }
            return false;
        }
    }
    si->second.emplace_back(std::move(v6_addr), duid, iaid, expire_time);
    return true;
}

const nk::ip_address &dynlease_query_refresh(const std::string &interface, const uint8_t *macaddr,
                                   int64_t expire_time)
{
    static nk::ip_address blank{};
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return blank;

    for (auto &i: si->second) {
        if (memcmp(&i.macaddr, macaddr, 6) == 0) {
            i.expire_time = expire_time;
            return i.addr;
        }
        return blank;
    }
    return blank;
}

const nk::ip_address &dynlease_query_refresh(const std::string &interface, const std::string &duid,
                                   uint32_t iaid, int64_t expire_time)
{
    static nk::ip_address blank{};
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return blank;

    for (auto &i: si->second) {
        if (i.duid == duid && i.iaid == iaid) {
            i.expire_time = expire_time;
            return i.addr;
        }
        return blank;
    }
    return blank;
}

bool dynlease_exists(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return false;

    for (auto &i: si->second) {
        if (i.addr == v4_addr && memcmp(&i.macaddr, macaddr, 6) == 0) {
            return get_current_ts() < i.expire_time;
        }
    }
    return false;
}

bool dynlease_exists(const std::string &interface, const nk::ip_address &v6_addr,
                     const std::string &duid, uint32_t iaid)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return false;

    for (auto &i: si->second) {
        if (i.addr == v6_addr && i.duid == duid && i.iaid == iaid) {
            return get_current_ts() < i.expire_time;
        }
    }
    return false;
}

bool dynlease_del(const std::string &interface, const nk::ip_address &v4_addr, const uint8_t *macaddr)
{
    auto si = dyn_leases_v4.find(interface);
    if (si == dyn_leases_v4.end()) return false;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == v4_addr && memcmp(&i->macaddr, macaddr, 6) == 0) {
            si->second.erase(i);
            return true;
        }
    }
    return false;
}

bool dynlease_del(const std::string &interface, const nk::ip_address &v6_addr,
                  const std::string &duid, uint32_t iaid)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return false;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == v6_addr && i->duid == duid && i->iaid == iaid) {
            si->second.erase(i);
            return true;
        }
    }
    return false;
}

bool dynlease_unused_addr(const std::string &interface, const nk::ip_address &addr)
{
    auto si = dyn_leases_v6.find(interface);
    if (si == dyn_leases_v6.end()) return true;

    const auto iend = si->second.end();
    for (auto i = si->second.begin(); i != iend; ++i) {
        if (i->addr == addr)
            return false;
    }
    return true;
}

// v4 <interface> <ip> <macaddr> <expire_time>
// v6 <interface> <ip> <duid> <iaid> <expire_time>

bool dynlease_serialize(const std::string &path)
{
    const auto tmp_path = path + ".tmp";
    const auto f = fopen(tmp_path.c_str(), "w");
    if (!f) {
        log_line("%s: failed to open '%s' for dynamic lease serialization\n",
                 __func__, path.c_str());
        return false;
    }
    SCOPE_EXIT{ fclose(f); unlink(tmp_path.c_str()); };
    for (const auto &i: dyn_leases_v4) {
        const auto &iface = i.first;
        const auto &ls = i.second;
        for (const auto &j: ls) {
            // Don't write out dynamic leases that have expired.
            if (get_current_ts() >= j.expire_time)
                continue;
            char wbuf[1024];
            int splen = snprintf(wbuf, sizeof wbuf, "v4 %s %s %2.x%2.x%2.x%2.x%2.x%2.x %zu\n",
                                 iface.c_str(), j.addr.to_string().c_str(),
                                 j.macaddr[0], j.macaddr[1], j.macaddr[2],
                                 j.macaddr[3], j.macaddr[4], j.macaddr[5], j.expire_time);
            if (splen < 0)
                suicide("%s: snprintf failed; return=%d", __func__, splen);
            if ((size_t)splen >= sizeof wbuf)
                suicide("%s: snprintf dest buffer too small %d >= %zu",
                        __func__, splen, sizeof wbuf);
            const auto fs = fwrite(wbuf, 1, splen, f);
            if (fs != (size_t)splen) {
                log_line("%s: short write %zd < %zu\n", __func__, fs, sizeof wbuf);
                return false;
            }
        }
    }
    for (const auto &i: dyn_leases_v6) {
        const auto &iface = i.first;
        const auto &ls = i.second;
        for (const auto &j: ls) {
            // Don't write out dynamic leases that have expired.
            if (get_current_ts() >= j.expire_time)
                continue;

            std::string wbuf;
            wbuf.append("v6 ");
            wbuf.append(iface);
            wbuf.append(" ");
            wbuf.append(j.addr.to_string());
            wbuf.append(" ");
            for (const auto &k: j.duid) {
                char tbuf[16];
                snprintf(tbuf, sizeof tbuf, "%.2hhx", k);
                wbuf.append(tbuf);
            }
            wbuf.append(" ");
            wbuf.append(std::to_string(j.iaid));
            wbuf.append(" ");
            wbuf.append(std::to_string(j.expire_time));
            wbuf.append("\n");
            const auto fs = fwrite(wbuf.c_str(), 1, wbuf.size(), f);
            if (fs != wbuf.size()) {
                log_line("%s: short write %zd < %zu", __func__, fs, wbuf.size());
                return false;
            }
        }
    }
    if (fflush(f)) {
        log_line("%s: fflush failed: %s", __func__, strerror(errno));
        return false;
    }
    const auto fd = fileno(f);
    if (fdatasync(fd)) {
        log_line("%s: fdatasync failed: %s", __func__, strerror(errno));
        return false;
    }
    if (rename(tmp_path.c_str(), path.c_str())) {
        log_line("%s: rename failed: %s", __func__, strerror(errno));
        return false;
    }
    return true;
}

// v4 <interface> <ip> <macaddr> <expire_time>
// v6 <interface> <ip> <duid> <iaid> <expire_time>

struct cfg_parse_state {
    cfg_parse_state() : st(nullptr), cs(0), parse_error(false) {}
    void newline() {
        duid.clear();
        macaddr.clear();
        v4_addr.clear();
        v6_addr.clear();
        interface.clear();
        iaid = 0;
        expire_time = 0;
        parse_error = false;
    }
    const char *st;
    int cs;

    std::string duid;
    std::string macaddr;
    std::string v4_addr;
    std::string v6_addr;
    std::string interface;
    int64_t expire_time;
    uint32_t iaid;
    bool parse_error;
};

static inline std::string lc_string(const char *s, size_t slen)
{
    auto r = std::string(s, slen);
    for (auto &i: r) i = tolower(i);
    return r;
}

%%{
    machine dynlease_line_m;
    access cps.;

    action St { cps.st = p; }

    action InterfaceEn { cps.interface = std::string(cps.st, p - cps.st); }
    action DuidEn { cps.duid = lc_string(cps.st, p - cps.st); }
    action IaidEn {
        if (auto t = nk::from_string<uint32_t>(lc_string(cps.st, p - cps.st))) cps.iaid = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }
    action MacAddrEn { cps.macaddr = lc_string(cps.st, p - cps.st); }
    action V4AddrEn { cps.v4_addr = lc_string(cps.st, p - cps.st); }
    action V6AddrEn { cps.v6_addr = lc_string(cps.st, p - cps.st); }
    action ExpireTimeEn {
        if (auto t = nk::from_string<int64_t>(std::string(cps.st, p - cps.st))) cps.expire_time = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }

    action V4EntryEn {
        emplace_dynlease_state(linenum, std::move(cps.interface), std::move(cps.v4_addr),
                               cps.macaddr, cps.expire_time);
    }
    action V6EntryEn {
        emplace_dynlease_state(linenum, std::move(cps.interface), std::move(cps.v6_addr),
                               std::move(cps.duid), cps.iaid, cps.expire_time);
    }

    interface = alnum+ >St %InterfaceEn;
    duid = (xdigit+ | (xdigit{2} ('-' xdigit{2})*)+) >St %DuidEn;
    iaid = digit+ >St %IaidEn;
    macaddr = ((xdigit{2} ':'){5} xdigit{2}) >St %MacAddrEn;
    v4_addr = (digit{1,3} | '.')+ >St %V4AddrEn;
    v6_addr = (xdigit{1,4} | ':')+ >St %V6AddrEn;
    expire_time = digit+ >St %ExpireTimeEn;

    v4_entry = space* 'v4' space+ interface space+ v4_addr space+ macaddr space+ expire_time space*;
    v6_entry = space* 'v6' space+ interface space+ v6_addr space+ duid space+ iaid space+ expire_time space*;

    main := v4_entry %V4EntryEn | v6_entry %V6EntryEn;
}%%

%% write data;

static int do_parse_dynlease_line(cfg_parse_state &cps, const char *p, size_t plen,
                             const size_t linenum)
{
    const char *pe = p + plen;
    const char *eof = pe;

    %% write init;
    %% write exec;

    if (cps.parse_error) return -1;
    if (cps.cs >= dynlease_line_m_first_final)
        return 1;
    if (cps.cs == dynlease_line_m_error)
        return -1;
    return -2;
}

bool dynlease_deserialize(const std::string &path)
{
    char buf[MAX_LINE];
    const auto f = fopen(path.c_str(), "r");
    if (!f) {
        log_line("%s: failed to open '%s' for dynamic lease deserialization\n",
                 __func__, path.c_str());
        return false;
    }
    SCOPE_EXIT{ fclose(f); };
    dyn_leases_v4.clear();
    dyn_leases_v6.clear();
    size_t linenum = 0;
    cfg_parse_state ps;
    while (!feof(f)) {
        if (!fgets(buf, sizeof buf, f)) {
            if (!feof(f))
                log_line("%s: io error fetching line of '%s'\n", __func__, path.c_str());
            break;
        }
        auto llen = strlen(buf);
        if (llen == 0)
            continue;
        if (buf[llen-1] == '\n')
            buf[--llen] = 0;
        ++linenum;
        ps.newline();
        const auto r = do_parse_dynlease_line(ps, buf, llen, linenum);
        if (r < 0) {
            if (r == -2)
                log_line("%s: Incomplete dynlease at line %zu; ignoring",
                         __func__, linenum);
            else
                log_line("%s: Malformed dynlease at line %zu; ignoring.",
                         __func__, linenum);
            continue;
        }
    }
    return true;
}


Remove useless unused variable in RubyParser

%%{
  machine message;

  getkey (chunk.getbyte(p) ^ 128) - 128;

  action mark {
    mark = p
  }
  action mark_key {
    mark_key = chunk.byteslice(mark, p - mark)
    mark = nil
  }
  action mark_message {
    mark_message = Stomp::Message.new(nil, nil)
    mark_message_size = 0
  }
  action check_message_size {
    mark_message_size += 1
    raise MessageSizeExceeded if mark_message_size > max_message_size
  }

  action write_command {
    mark_message.write_command(chunk.byteslice(mark, p - mark))
    mark = nil
  }

  action write_header {
    mark_message.write_header(mark_key, chunk.byteslice(mark, p - mark))
    mark_key = mark = nil
  }

  action write_body {
    mark_message.write_body(chunk.byteslice(mark, p - mark))
    mark = nil
  }

  action finish_headers {
    mark_content_length = mark_message.content_length
  }

  action consume_null {
    (p - mark) < mark_content_length if mark_content_length
  }

  action consume_octet {
    if mark_content_length
      (p - mark) < mark_content_length
    else
      true
    end
  }

  action finish_message {
    yield mark_message
    mark_message = nil
  }

  include message_common "parser_common.rl";
}%%

module Stompede
  module Stomp
    class RubyParser
      class State
        def initialize
          @cs = RubyParser.start
          @chunk = nil
          @mark = nil
          @mark_key = nil
          @mark_message = nil
          @mark_message_size = nil
          @mark_content_length = nil
        end

        # You want documentation? HAHA.
        attr_accessor :chunk
        attr_accessor :cs
        attr_accessor :mark
        attr_accessor :mark_key
        attr_accessor :mark_message
        attr_accessor :mark_message_size
        attr_accessor :mark_content_length
      end

      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] chunk
      # @param [State] state previous parser state, or nil for initial state
      # @param [Integer] max_message_size
      # @yield [message] yields each message as it is parsed
      # @yieldparam message [Stomp::Message]
      def self._parse(chunk, state, max_message_size)
        if state.chunk
          p = state.chunk.bytesize
          chunk = state.chunk << chunk
        else
          p = 0
        end

        pe = chunk.bytesize # special

        cs = state.cs
        mark = state.mark
        mark_key = state.mark_key
        mark_message = state.mark_message
        mark_message_size = state.mark_message_size
        mark_content_length = state.mark_content_length

        %% write exec;

        if mark
          state.chunk = chunk
        else
          state.chunk = nil
        end

        state.cs = cs
        state.mark = mark
        state.mark_key = mark_key
        state.mark_message = mark_message
        state.mark_message_size = mark_message_size
        state.mark_content_length = mark_content_length

        if cs == RubyParser.error
          Stomp.build_parse_error(chunk, p)
        else
          nil
        end
      end

      def initialize(max_message_size = Stomp.max_message_size)
        @state = State.new
        @max_message_size = Integer(max_message_size)
      end

      # Parse a chunk.
      #
      # @param [String] chunk
      # @yield [message]
      # @yieldparam [Stomp::Message] message
      def parse(chunk)
        @error ||= self.class._parse(chunk, @state, @max_message_size) do |message|
          yield message
        end

        raise @error if @error
      end
    end
  end
end

%%{
  machine message;

  getkey (chunk.getbyte(p) ^ 128) - 128;

  action mark {
    mark = p
  }
  action mark_key {
    mark_key = chunk.byteslice(mark, p - mark)
    mark = nil
  }
  action mark_message {
    mark_message = Stomp::Message.new(nil, nil)
    mark_message_size = 0
  }
  action check_message_size {
    mark_message_size += 1
    raise MessageSizeExceeded if mark_message_size > max_message_size
  }

  action write_command {
    mark_message.write_command(chunk.byteslice(mark, p - mark))
    mark = nil
  }

  action write_header {
    mark_message.write_header(mark_key, chunk.byteslice(mark, p - mark))
    mark_key = mark = nil
  }

  action write_body {
    mark_message.write_body(chunk.byteslice(mark, p - mark))
    mark = nil
  }

  action finish_headers {
    mark_content_length = mark_message.content_length
  }

  action consume_null {
    (p - mark) < mark_content_length if mark_content_length
  }

  action consume_octet {
    if mark_content_length
      (p - mark) < mark_content_length
    else
      true
    end
  }

  action finish_message {
    yield mark_message
    mark_message = nil
  }

  include message_common "parser_common.rl";
}%%

module Stompede
  module Stomp
    class RubyParser
      class State
        def initialize
          @p = 0
          @cs = RubyParser.start
          @chunk = nil
          @mark = nil
          @mark_key = nil
          @mark_message = nil
          @mark_message_size = nil
          @mark_content_length = nil
        end

        # You want documentation? HAHA.
        attr_accessor :chunk
        attr_accessor :p
        attr_accessor :cs
        attr_accessor :mark
        attr_accessor :mark_key
        attr_accessor :mark_message
        attr_accessor :mark_message_size
        attr_accessor :mark_content_length
      end

      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] chunk
      # @param [State] state previous parser state, or nil for initial state
      # @param [Integer] max_message_size
      # @yield [message] yields each message as it is parsed
      # @yieldparam message [Stomp::Message]
      def self._parse(chunk, state, max_message_size)
        if state.chunk
          p = state.chunk.bytesize
          chunk = state.chunk << chunk
        else
          p = 0
        end

        pe = chunk.bytesize # special

        cs = state.cs
        mark = state.mark
        mark_key = state.mark_key
        mark_message = state.mark_message
        mark_message_size = state.mark_message_size
        mark_content_length = state.mark_content_length

        %% write exec;

        if mark
          state.p = chunk.bytesize
          state.chunk = chunk
        else
          state.p = 0
          state.chunk = nil
        end

        state.cs = cs
        state.mark = mark
        state.mark_key = mark_key
        state.mark_message = mark_message
        state.mark_message_size = mark_message_size
        state.mark_content_length = mark_content_length

        if cs == RubyParser.error
          Stomp.build_parse_error(chunk, p)
        else
          nil
        end
      end

      def initialize(max_message_size = Stomp.max_message_size)
        @state = State.new
        @max_message_size = Integer(max_message_size)
      end

      # Parse a chunk.
      #
      # @param [String] chunk
      # @yield [message]
      # @yieldparam [Stomp::Message] message
      def parse(chunk)
        @error ||= self.class._parse(chunk, @state, @max_message_size) do |message|
          yield message
        end

        raise @error if @error
      end
    end
  end
end