Ragel: optional debug

Ragel: optional debug

# Ragel lexer for VCF-header
#
# This is a partial lexer for the VCF header format. Bio-vcf uses this
# to generate meta information in (for example) JSON format. The
# advantage of using a full state engine is that it allows for easy
# parsing of key-value pairs with syntax checking and, for example,
# escaped quotes in quoted string values. This edition validates ID and
# Number fields only.

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  do_debug = (options[:debug] == true)
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if do_debug
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res if do_debug
  res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  p VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Ragel lexer for VCF-header
#
# This is a partial lexer for the VCF header format. Bio-vcf uses this
# to generate meta information in (for example) JSON format. The
# advantage of using a full state engine is that it allows for easy
# parsing of key-value pairs with syntax checking and, for example,
# escaped quotes in quoted string values. This edition validates ID and
# Number fields only.

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Added single-quoted named group pattern

Added single-quoted named group pattern

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';

  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end

      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'L';  self.emit(type, :letter_any,       text, ts, te)
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'M';  self.emit(type, :mark_any,         text, ts, te)
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'N';  self.emit(type, :number_any,       text, ts, te)
      when 'Nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'Nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'No'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'P';  self.emit(type, :punct_any,        text, ts, te)
      when 'Pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'Ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'Pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'Pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'Po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 'S';  self.emit(type, :symbol_any,       text, ts, te)
      when 'Sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'So'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'Z';  self.emit(type, :separator_any,        text, ts, te)
      when 'Zs'; self.emit(type, :separator_space,      text, ts, te)
      when 'Zl'; self.emit(type, :separator_line,       text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph,  text, ts, te)

      # Codepoints
      when 'C';  self.emit(type, :code_any,         text, ts, te)
      when 'Cc'; self.emit(type, :code_control,     text, ts, te)
      when 'Cf'; self.emit(type, :code_format,      text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate,   text, ts, te)
      when 'Co'; self.emit(type, :code_private,     text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned,  text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    wild {
      self.emit(:meta, :wild, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when /\(\?<\w+>/
        self.emit(:group, :named, text, ts, te)
      end
    };

    group_open  {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end

      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'L';  self.emit(type, :letter_any,       text, ts, te)
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'M';  self.emit(type, :mark_any,         text, ts, te)
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'N';  self.emit(type, :number_any,       text, ts, te)
      when 'Nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'Nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'No'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'P';  self.emit(type, :punct_any,        text, ts, te)
      when 'Pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'Ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'Pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'Pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'Po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 'S';  self.emit(type, :symbol_any,       text, ts, te)
      when 'Sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'So'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'Z';  self.emit(type, :separator_any,        text, ts, te)
      when 'Zs'; self.emit(type, :separator_space,      text, ts, te)
      when 'Zl'; self.emit(type, :separator_line,       text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph,  text, ts, te)

      # Codepoints
      when 'C';  self.emit(type, :code_any,         text, ts, te)
      when 'Cc'; self.emit(type, :code_control,     text, ts, te)
      when 'Cf'; self.emit(type, :code_format,      text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate,   text, ts, te)
      when 'Co'; self.emit(type, :code_private,     text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned,  text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    wild {
      self.emit(:meta, :wild, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when /\(\?<\w+>/
        self.emit(:group, :named, text, ts, te)
      end
    };

    group_open  {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%


End list should only occur in list context.

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_KEYGROUP);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_keygroup->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_keygroup->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the keygroup
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the keygroup */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_keygroup {
		char *ancestor, *tofree, *keygroupname;

		struct toml_node *place = toml_root;

		tofree = keygroupname = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&keygroupname, "."))) {
			struct toml_keygroup_item *item;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_KEYGROUP;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		free(tofree);

		if (place->type != TOML_KEYGROUP) {
			parse_error = 1;
			fbreak;
		}

		cur_keygroup = place;
	}

	lines = (
		start: (
			# count the indentation to know where the keygroups end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @{curline++;} ->start ),

		# a keygroup
		keygroup: ( name ']' @saw_keygroup ->start ),

		# the boolean data type
		true: 	( any	>{fhold;} $saw_bool	->start ),
		false: 	( any 	>{fhold;} $saw_bool	->start ),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		string: (
			'"'  $saw_string				->start			|
			[\n] ${curline++; *strp++=fc;}	->string		|
			[\\]							->str_escape	|
			[^"\n\\] ${*strp++=fc;}			->string
		),
		str_escape: (
			"0"	${*strp++=0;}		-> string |
			"t"	${*strp++='\t';}	-> string |
			"n"	${*strp++='\n';}	-> string |
			"r"	${*strp++='\r';}	-> string |
			[^0tnr]	${*strp++=fc;}	-> string
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true		|
			'false'	@{number = 0;}				-> false	|
			'"' ${strp = string;}				-> string	|
			('-'|'+') ${fhold;number = 0;}		-> sign		|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->val		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->keygroup	|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	int parse_error = 0, malloc_error = 0;

	struct toml_node *cur_keygroup = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (cs == toml_error || parse_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include <time.h>

#include "toml.h"
#include <signal.h>

struct toml_stack_item {
	struct list_node list;
	enum toml_type list_type;
	struct toml_node *node;
};

static const char *
toml_type_to_str(enum toml_type type)
{
#define CASE_ENUM_TO_STR(x) case(x): return #x
	switch (type) {
	CASE_ENUM_TO_STR(TOML_ROOT);
	CASE_ENUM_TO_STR(TOML_KEYGROUP);
	CASE_ENUM_TO_STR(TOML_LIST);
	CASE_ENUM_TO_STR(TOML_INT);
	CASE_ENUM_TO_STR(TOML_FLOAT);
	CASE_ENUM_TO_STR(TOML_STRING);
	CASE_ENUM_TO_STR(TOML_DATE);
	CASE_ENUM_TO_STR(TOML_BOOLEAN);
	}
#undef CASE_ENUM_TO_STR
	return "unknown toml type";
}

%%{
	machine toml;

	whitespace = [\t ]*;

	name = (print - (whitespace|']'))+ >{ts = p;};

	action float_add_place {
		floating += (fc - '0') * ((float)1/dec_pos);
		dec_pos *= 10;
	}

	action saw_key {
		name = strndup(ts, namelen);
	}

	action saw_bool {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_BOOLEAN) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_BOOLEAN), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_BOOLEAN;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_BOOLEAN;
			item->node.value.integer = number;

			list_add_tail(&cur_keygroup->value.map, &item->map);
		}
	}

	action saw_int {
		number *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_INT) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_INT), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_INT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_INT;
			item->node.value.integer = number;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.name = name;
			item->node.type = TOML_INT;
			item->node.value.integer = number;

			list_add_tail(&cur_keygroup->value.map, &item->map);
		}

		fhold;
	}

	action saw_float {
		floating *= sign;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_FLOAT) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_FLOAT), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_FLOAT;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_FLOAT;
			item->node.value.floating = floating;
		}

		fhold;
	}

	action saw_string {
		int len = strp - string;
		*strp = 0;

		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_STRING) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_STRING), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_STRING;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_STRING;
			item->node.value.string = malloc(len);
			if (!item->node.value.string) {
				malloc_error = 1;
				fbreak;
			}
			memcpy(item->node.value.string, string, len);
		}
	}

	action saw_date {
		struct toml_stack_item *cur_list =
						list_tail(&list_stack, struct toml_stack_item, list);

		if (cur_list) {
			if (cur_list->list_type && cur_list->list_type != TOML_DATE) {
				fprintf(stderr, "incompatible types list %s this %s line %d\n",
						toml_type_to_str(cur_list->list_type),
						toml_type_to_str(TOML_DATE), curline);
				parse_error = 1;
				fbreak;
			}
			cur_list->list_type = TOML_DATE;

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
			item->node.name = NULL;

			list_add_tail(&cur_list->node->value.list, &item->list);

			fnext list;
		} else {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			item->node.type = TOML_DATE;
			item->node.value.epoch = timegm(&tm);
		}
	}

	action start_list {
		struct toml_node *node;

		/*
		 * if the list stack is empty add this list to the keygroup
		 * otherwise it should be added the to list on top of the stack
		 */
		if (list_empty(&list_stack)) {
			struct toml_keygroup_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			/* first add it to the keygroup */
			item->node.type = TOML_LIST;
			list_head_init(&item->node.value.list);
			list_add_tail(&cur_keygroup->value.map, &item->map);
			item->node.name = name;
			node = &item->node;
		} else {
			struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

			struct toml_list_item *item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}

			item->node.type = TOML_LIST;
			item->node.name = NULL;
			list_head_init(&item->node.value.list);

			list_add_tail(&tail->node->value.list, &item->list);
			node = &item->node;
		}

		/* push this list onto the stack */
		struct toml_stack_item *stack_item = malloc(sizeof(*stack_item));
		if (!stack_item) {
			malloc_error = 1;
			fbreak;
		}
		stack_item->node = node;
		stack_item->list_type = 0;
		list_add_tail(&list_stack, &stack_item->list);
	}

	action end_list {
		struct toml_stack_item *tail =
						list_tail(&list_stack, struct toml_stack_item, list);

		list_del(&tail->list);
		free(tail);

		if (!list_empty(&list_stack))
			fnext list;
	}

	action saw_keygroup {
		char *ancestor, *tofree, *keygroupname;

		struct toml_node *place = toml_root;

		tofree = keygroupname = strndup(ts, (int)(p-ts));

		while ((ancestor = strsep(&keygroupname, "."))) {
			struct toml_keygroup_item *item;
			int found = 0;

			list_for_each(&place->value.map, item, map) {
				if (strcmp(item->node.name, ancestor) == 0) {
					place = &item->node;
					found = 1;
					break;
				}
			}

			if (found)
				continue;

			/* this is the auto-vivification */
			item = malloc(sizeof(*item));
			if (!item) {
				malloc_error = 1;
				fbreak;
			}
			item->node.name = strdup(ancestor);
			item->node.type = TOML_KEYGROUP;
			list_head_init(&item->node.value.map);
			list_add_tail(&place->value.map, &item->map);

			place = &item->node;
		}

		free(tofree);

		if (place->type != TOML_KEYGROUP) {
			parse_error = 1;
			fbreak;
		}

		cur_keygroup = place;
	}

	lines = (
		start: (
			# count the indentation to know where the keygroups end
			[\t ]* >{ indent = 0; } ${ indent++; } ->text
		),

		# just discard everything until newline
		comment: ( [^\n]*[\n] @{curline++;} ->start ),

		# a keygroup
		keygroup: ( name ']' @saw_keygroup ->start ),

		# the boolean data type
		true: 	( any	>{fhold;} $saw_bool	->start ),
		false: 	( any 	>{fhold;} $saw_bool	->start ),

		# String, we have to escape \0, \t, \n, \r, everything else can
		# be prefixed with a slash and the slash just gets dropped
		string: (
			'"'  $saw_string				->start			|
			[\n] ${curline++; *strp++=fc;}	->string		|
			[\\]							->str_escape	|
			[^"\n\\] ${*strp++=fc;}			->string
		),
		str_escape: (
			"0"	${*strp++=0;}		-> string |
			"t"	${*strp++='\t';}	-> string |
			"n"	${*strp++='\n';}	-> string |
			"r"	${*strp++='\r';}	-> string |
			[^0tnr]	${*strp++=fc;}	-> string
		),

		# A sign can optiionally prefix a number
		sign: (
			'-' ${sign = -1;}	->number |
			'+' ${sign = 1;}	->number
		),
		number: (
			digit ${number *= 10; number += fc - '0';}	->number			|
			'.'	${floating = number; dec_pos = 10;}		->fractional_part	|
			[\n] ${curline++;} $saw_int					->start				|
			[^0-9.]	$saw_int							->start
		),

		# When we don't know yet if this is going to be a date or a number
		# this is the state
		number_or_date: (
			digit ${number *= 10; number += fc - '0';}	->number_or_date	|
			'-'	${tm.tm_year = number - 1900;}			->date				|
			'.'	${floating = number * sign; dec_pos = 10;}			->fractional_part	|
			[\n] ${curline++;} $saw_int								->start	|
			[\t ,\]] $saw_int										->start
		),

		# Fractional part of a double
		fractional_part: (
			[0-9] 	$float_add_place	->fractional_part |
			[^0-9]  $saw_float			->start
		),

		# Zulu date, we've already picked up the first four digits and the '-'
		# when figuring this was a date and not a number
		date: (
			digit{2} @{tm.tm_mon = atoi(fpc-1) - 1;}
			'-'
			digit{2} @{tm.tm_mday = atoi(fpc-1);}
			'T'
			digit{2} @{tm.tm_hour = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_min = atoi(fpc-1);}
			':'
			digit{2} @{tm.tm_sec = atoi(fpc-1);}
			'Z' @saw_date ->start
		),

		# Non-list value
		singular: (
			'true'	@{number = 1;}				-> true		|
			'false'	@{number = 0;}				-> false	|
			'"' ${strp = string;}				-> string	|
			('-'|'+') ${fhold;number = 0;}		-> sign		|
			digit ${sign = 1;fhold;number = 0;}	-> number_or_date
		),

		# A list of values
		list: (
			'#'								->comment	|
			'\n' ${ curline++; }			->list		|
			[\t ]							->list		|
			','								->val		|
			']'	$end_list					->start		|
			[^#\t, \n\]] ${fhold;}			->val
		),

		# A val can be either a list or a singular value
		val: (
			'#'							->comment	|
			'\n' ${ curline++; }		->val		|
			[\t ]						->val		|
			'[' $start_list				->list		|
			']'	$end_list				->start		|
			[^#\t \n[\]] ${fhold;}		->singular
		)

		# A regular key
		key: (
			name  whitespace >{namelen = (int)(p-ts);} '=' $saw_key ->val
		),

		# Text stripped of leading whitespace
		text: (
			'#'							->comment	|
			'['							->keygroup	|
			[\t ]						->text		|
			'\n' ${curline++;}			->start     |
			']' $end_list				->start		|
			[^#[\t \n,\]]	${fhold;}	->key
		)
	);

	main := lines?;
}%%

%%write data;

int
toml_parse(struct toml_node *toml_root, char *buf, int buflen)
{
	int indent = 0, cs, curline = 1;
	char *p, *pe;
	char *ts;
	char string[1024], *strp;
	int sign, number, dec_pos, namelen;
	struct tm tm;
	double floating;
	char *name;
	int parse_error = 0, malloc_error = 0;

	struct toml_node *cur_keygroup = toml_root;

	struct list_head list_stack;
	list_head_init(&list_stack);

	assert(toml_root->type == TOML_ROOT);

	%% write init;

	p = buf;
	pe = buf + buflen;

	%% write exec;

	if (malloc_error) {
		fprintf(stderr, "malloc failed, line %d\n", curline);
		return 1;
	}

	if (cs == toml_error || parse_error) {
		fprintf(stderr, "PARSE_ERROR, line %d, p = '%.5s'", curline, p);
		return 1;
	}

	return 0;
}


Fixed reseting byte accumulator in pack Bb.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static String* string(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_s"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    typedef String* (*EncodingCoerce)(STATE, CallFrame*, Object*);

    inline static String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          EncodingCoerce coerce)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      return coerce(state, call_frame, obj);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline static void swapf(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap32(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline static void swapd(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap64(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    static void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    inline static size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    static void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    static void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline static size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    static void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    static void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(size() * 4);

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static String* string(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_s"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    typedef String* (*EncodingCoerce)(STATE, CallFrame*, Object*);

    inline static String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          EncodingCoerce coerce)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      return coerce(state, call_frame, obj);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline static void swapf(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap32(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline static void swapd(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap64(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    static void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    inline static size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    static void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    static void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline static size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    static void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    static void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(size() * 4);

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


add comments on the files purpose

add comments on the files purpose