Fixed quantified literals when longer than one character long

%%{
  machine re_scanner;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name_unicode = 'alnum'i | 'alpha'i | 'any'i   | 'ascii'i | 'blank'i |
                          'cntrl'i | 'digit'i | 'graph'i | 'lower'i | 'print'i |
                          'punct'i | 'space'i | 'upper'i | 'word'i  | 'xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # TODO: in what context?
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts-1..te-1].pack('c*')

      # TODO: rename :nonproperty to :nonproperty
      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
      fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open  {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences. TODO: verify
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      if (te - ts) > 1 and data[te]
        case data[te].chr
        when '?', '*', '+', '{'
          p -= 1
          self.emit(:literal, :literal, data[ts..te-2].pack('c*'), ts, te)
        else
          self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
        end
      else
        self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
      end
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and:
  #
  # If a block is given, calls it for each emitted token, and returns nil
  # when done.
  #
  # If no block is given, collect the emitted arguments (as an array) into
  # an array and return that when done.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @block  = block_given? ? block : nil
    @tokens = block_given? ? nil : []

    %% write init;
    %% write exec;

    @tokens
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @block
      @block.call type, token, text, ts, te
    else
      @tokens << [type, token, text, ts, te]
    end
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name_unicode = 'Alnum'i | 'Alpha'i | 'Any'i   | 'Ascii'i | 'Blank'i |
                          'Cntrl'i | 'Digit'i | 'Graph'i | 'Lower'i | 'Print'i |
                          'Punct'i | 'Space'i | 'Upper'i | 'Word'i | 'Xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # TODO: in what context?
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts-1..te-1].pack('c*')

      # TODO: rename :nonproperty to :nonproperty
      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
      fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #     .       any
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open  {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences. TODO: verify
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and:
  #
  # If a block is given, calls it for each emitted token, and returns nil
  # when done.
  #
  # If no block is given, collect the emitted arguments (as an array) into
  # an array and return that when done.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @block  = block_given? ? block : nil
    @tokens = block_given? ? nil : []

    %% write init;
    %% write exec;

    @tokens
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @block
      @block.call type, token, text, ts, te
    else
      @tokens << [type, token, text, ts, te]
    end
  end

end # module Regexp::Lexer


Fixed bug in config file parser.

Fixed bug in config file parser.

#include "configfile.hpp"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <err.h>
#include <stdarg.h>

#define MAX_WARNINGS 10

#define WARN(...) \
    do { \
        configFileWarn(path, lineno, __VA_ARGS__); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

static void configFileWarn(char const *path, unsigned int lineno,
                           char const *msg, ...)
{
    va_list ap;
    fprintf(stderr, "%s:%u: ", path, lineno);
    va_start(ap, msg);
    vfprintf(stderr, msg, ap);
    va_end(ap);
    putc('\n', stderr);
}

#define WARNC(msg, c) \
    do { \
        configFileWarnC(path, lineno, msg, *p); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

%%{
    machine configfile;
    write data;

    action putc {
        tmp[tmppos++] = *p;
    }
    action pute {
        switch (*p) {
        case 'n': tc = '\n'; break;
        case '\\': tc = '\\'; break;
        case '"': tc = '"'; break;
        default:
            tc = *p;
            WARN("Invalid escape sequence: \"\\%c\"", (int)tc);
            break;
        }
        tmp[tmppos++] = tc;
    }
    action startsec {
        secend = 0;
        keyend = 0;
        valend = 0;
        tmppos = 0;
    }
    action startvar {
        keyend = 0;
        valend = 0;
        tmppos = secend;
        quoted = false;
    }
    action setquoted {
        quoted = true;
    }
    action endsec {
        secend = tmppos;
    }
    action endkey {
        keyend = tmppos;
    }
    action endval {
        valend = tmppos;
    }
    action nextline {
        lineno += 1;
    }
    action endvarline {
        if (!secend)
            WARN("Variable appears outside config section");
        else {
            if (!quoted) {
                while (valend > keyend && tmp[valend - 1] == ' ')
                    valend--;
            }
            char *d = alloc(valend + 3),
                *sec = d, *key = d + secend + 1, *val = d + keyend + 2;
            memcpy(sec, tmp, secend);
            sec[secend] = '\0';
            memcpy(key, tmp + secend, keyend - secend);
            key[keyend - secend] = '\0';
            memcpy(val, tmp + keyend, valend - keyend);
            val[valend - keyend] = '\0';
            Var v = { sec, key, val };
            vars_.push_back(v);
            sorted_ = false;
        }
    }
    action error {
        WARN("Syntax error in configuration file");
        fhold;
        fgoto eaterr;
    }
    action resume {
        fhold;
        fgoto main;
    }

    lbreak = (('\r' '\n'?) | '\n') $(line,1) %(line,0) >nextline ;
    ws = (' ' | '\t')* ;
    comment = (';' | '#') (any - ('\n' | '\r'))* ;

    csec = (alnum | '.' | '-')+ @putc %endsec ;
    ckey = (alnum | '.' | '-')+ @putc %endkey ;
    cvunquotc = print - ('"' | '\\' | '#' | ';') ;
    cvunquot = ((cvunquotc - ' ') cvunquotc**) @putc ;
    cvescape = '\\' print %to(pute);
    cvplain = (print - ('"' | '\\')) @putc ;
    cvquotc = cvplain | cvescape ;
    cvquot = ('"' cvquotc* '"') >setquoted ;
    cval = (cvunquot | cvquot)? %endval ;

    varline = (ckey ws '=' ws cval ws) >startvar %endvarline ;
    secline = ('[' ws csec ws ']' ws) >startsec ;

    line = ws (varline | secline)? comment? ;

    main := (line lbreak)* line? $!error ;

    eaterr := (any - ('\n' | '\r'))* $!resume ;
}%%

void ConfigFile::read(char const *path)
{
    unsigned int warncount = 0;
    size_t bufsz = 1024, tmpsz = bufsz * 2;
    char *buf = 0, *tmp = 0;
    FILE *f = 0;
    f = fopen(path, "rb");
    if (!f) goto error;
    buf = reinterpret_cast<char *>(malloc(bufsz));
    if (!buf) goto error;
    tmp = reinterpret_cast<char *>(malloc(tmpsz));
    if (!tmp) goto error;
    try {
        int cs;
        size_t tmppos = 0, secend = 0, keyend = 0, valend = 0, amt;
        unsigned int lineno = 1;
        unsigned char tc;
        bool quoted = false;
        %% write init;
        do {
            amt = fread(buf, 1, sizeof(buf), f);
            char *p = buf, *pe = buf + amt, *eof = NULL;
            if (!amt) {
                if (ferror(f))
                    goto error;
                eof = pe;
            }
            if (tmppos + amt > tmpsz) {
                fprintf(stderr, "%s:%u: Line too long\n", path, lineno);
                break;
            }
            %% write exec;
        } while (amt);
    } catch (...) {
        free(tmp);
        free(buf);
        fclose(f);
        throw;
    }
    goto done;
error:
    warn("Error reading config file '%s'", path);
done:
    if (warncount == MAX_WARNINGS)
        fprintf(stderr, "%s: Too many warnings, aborting\n", path);
    free(tmp);
    free(buf);
    if (f) fclose(f);
    return;
}

#include "configfile.hpp"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <err.h>
#include <stdarg.h>

#define MAX_WARNINGS 10

#define WARN(...) \
    do { \
        configFileWarn(path, lineno, __VA_ARGS__); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

static void configFileWarn(char const *path, unsigned int lineno,
                           char const *msg, ...)
{
    va_list ap;
    fprintf(stderr, "%s:%u: ", path, lineno);
    va_start(ap, msg);
    vfprintf(stderr, msg, ap);
    va_end(ap);
    putc('\n', stderr);
}

#define WARNC(msg, c) \
    do { \
        configFileWarnC(path, lineno, msg, *p); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

%%{
    machine configfile;
    write data;

    action putc {
        tmp[tmppos++] = *p;
    }
    action pute {
        switch (*p) {
        case 'n': tc = '\n'; break;
        case '\\': tc = '\\'; break;
        case '"': tc = '"'; break;
        default:
            tc = *p;
            WARN("Invalid escape sequence: \"\\%c\"", (int)tc);
            break;
        }
        tmp[tmppos++] = tc;
    }
    action startsec {
        secend = 0;
        keyend = 0;
        valend = 0;
        tmppos = 0;
    }
    action startvar {
        keyend = 0;
        valend = 0;
        tmppos = secend;
        quoted = false;
    }
    action setquoted {
        quoted = true;
    }
    action endsec {
        secend = tmppos;
    }
    action endkey {
        keyend = tmppos;
    }
    action endval {
        valend = tmppos;
    }
    action nextline {
        lineno += 1;
    }
    action endvarline {
        if (!secend)
            WARN("Variable appears outside config section");
        else {
            if (!quoted) {
                while (valend > keyend && tmp[valend - 1] == ' ')
                    valend--;
            }
            char *d = alloc(valend + 3),
                *sec = d, *key = d + secend + 1, *val = d + keyend + 2;
            memcpy(sec, tmp, secend);
            sec[secend] = '\0';
            memcpy(key, tmp + secend, keyend - secend);
            key[keyend - secend] = '\0';
            memcpy(val, tmp + keyend, valend - keyend);
            val[valend - keyend] = '\0';
            Var v = { sec, key, val };
            vars_.push_back(v);
        }
    }
    action error {
        WARN("Syntax error in configuration file");
        fhold;
        fgoto eaterr;
    }
    action resume {
        fhold;
        fgoto main;
    }

    lbreak = (('\r' '\n'?) | '\n') $(line,1) %(line,0) >nextline ;
    ws = (' ' | '\t')* ;
    comment = (';' | '#') (any - ('\n' | '\r'))* ;

    csec = (alnum | '.' | '-')+ @putc %endsec ;
    ckey = (alnum | '.' | '-')+ @putc %endkey ;
    cvunquotc = print - ('"' | '\\' | '#' | ';') ;
    cvunquot = ((cvunquotc - ' ') cvunquotc**) @putc ;
    cvescape = '\\' print %to(pute);
    cvplain = (print - ('"' | '\\')) @putc ;
    cvquotc = cvplain | cvescape ;
    cvquot = ('"' cvquotc* '"') >setquoted ;
    cval = (cvunquot | cvquot)? %endval ;

    varline = (ckey ws '=' ws cval ws) >startvar %endvarline ;
    secline = ('[' ws csec ws ']' ws) >startsec ;

    line = ws (varline | secline)? comment? ;

    main := (line lbreak)* line? $!error ;

    eaterr := (any - ('\n' | '\r'))* $!resume ;
}%%

void ConfigFile::read(char const *path)
{
    unsigned int warncount = 0;
    size_t bufsz = 1024, tmpsz = bufsz * 2;
    char *buf = 0, *tmp = 0;
    FILE *f = 0;
    f = fopen(path, "rb");
    if (!f) goto error;
    buf = reinterpret_cast<char *>(malloc(bufsz));
    if (!buf) goto error;
    tmp = reinterpret_cast<char *>(malloc(tmpsz));
    if (!tmp) goto error;
    try {
        int cs;
        size_t tmppos = 0, secend = 0, keyend = 0, valend = 0, amt;
        unsigned int lineno = 1;
        unsigned char tc;
        bool quoted = false;
        %% write init;
        do {
            amt = fread(buf, 1, sizeof(buf), f);
            char *p = buf, *pe = buf + amt, *eof = NULL;
            if (!amt) {
                if (ferror(f))
                    goto error;
                eof = pe;
            }
            if (tmppos + amt > tmpsz) {
                fprintf(stderr, "%s:%u: Line too long\n", path, lineno);
                break;
            }
            %% write exec;
        } while (amt);
    } catch (...) {
        free(tmp);
        free(buf);
        fclose(f);
        throw;
    }
    goto done;
error:
    warn("Error reading config file '%s'", path);
done:
    if (warncount == MAX_WARNINGS)
        fprintf(stderr, "%s: Too many warnings, aborting\n", path);
    free(tmp);
    free(buf);
    if (f) fclose(f);
    return;
}


Use a better scanner for the picture elements

Try and capture everything in a single pass rather than 4 passes.

package handbrake

import (
	"fmt"
	"strings"
	"strconv"
)

%%{
	machine handbrake;
	write data;
}%%

type Section int
const (
	NONE Section = iota
	CHAPTER
	AUDIO
	SUBTITLE
)

func parseTime(timestring string) float64 {
	rawTime := strings.Trim(timestring, " \n");
	splitTime := strings.Split(rawTime, ":")
	var length float64
	hours, err := strconv.ParseInt(splitTime[0], 10, 8)
	if err != nil {
		panic(err)
	}
	length += float64(hours * 60 * 60)
	minutes, err := strconv.ParseInt(splitTime[1], 10, 8)
	length += float64(minutes * 60)
	seconds, err := strconv.ParseInt(splitTime[2], 10, 8)
	length += float64(seconds)
	return length
}

func parseOutput(data string) HandBrakeMeta {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	var stack = []int{0}
	var section = NONE
	var meta = HandBrakeMeta{}
	line := 1
	fmt.Printf("%02d: ", line)
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		stitle := |*
			([^.])+[.]alnum+ => {
				meta.Title = strings.Trim(data[ts:te], " \n");
				fmt.Printf("%s", meta.Title);
				fret;
			};
		*|;
		sduration := |*
			space+;
			digit{2}[:]digit{2}[:]digit{2} => {
				meta.Duration = parseTime(data[ts:te])
				fmt.Printf("%f", meta.Duration);
				fret;
			};
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} "," => { fmt.Printf("1-%s:", data[ts:te-1]); };
			space*;
			"pixel" space+ "aspect:";
			digit{1,4} "/" digit{1,4} "," => { fmt.Printf("2-%s:", data[ts:te-1]); };
			space*;
			"display" space+ "aspect:";
			space*;
			digit . "." . digit{1,3} "," => { fmt.Printf("3-%s:", data[ts:te-1]); };
			space*;
			digit{2} . "." digit{3} space+ "fps" "\n" => { fmt.Printf("4-%s", data[ts:te-5]); p -= 1; fret; };
		*|;
		crop := |*
			space*;
			digit{1,3} "/" digit{1,3} "/" digit{1,3} "/" digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		atrack := |*
# Language
			[A-Z] alpha+ => { fmt.Printf("a-%s:", data[ts:te]); };
			space;
			"(";
# Codec
			"AC3" | "DTS" | "aac" => { fmt.Printf("b-%s:", data[ts:te]); };
			")";
			space;
			"(";
# Channels
			digit . "." . digit => { fmt.Printf("c-%s:", data[ts:te]) };
			space;
			"ch) ";
# Ignore this bit
			"(iso" digit{3} "-" digit ":" space lower{3} "),";
# Hertz
			space;
			digit+ "Hz," => { fmt.Printf("d-%s:", data[ts:te-3]) };
			space;
# Bps
			digit+ "bps" => { fmt.Printf("e-%s", data[ts:te-3]); fret; };
			*|;
		subtype = "Bitmap" | "Text";
		format = "VOBSUB" | "UTF-8";
		subtitle := |*
			[A-Z] alpha+ - subtype - format => { fmt.Printf("a-%s:", data[ts:te]); };
			space;
			"(iso" digit{3} "-" digit ":" space lower{3} ")";
			space;
			"(" subtype ")" => { fmt.Printf("b-%s:", data[ts+1:te-1]); };
			"(" format ")" => { fmt.Printf("c-%s", data[ts+1:te-1]); fret; };
		*|;
		word = [a-z]+;
		prefix = space+ "+";
		prefixsp = prefix space;
		stream = prefixsp "stream:";
		duration = prefixsp "duration:";
		video = prefixsp "size:";
		autocrop = prefixsp "autocrop:";
		track = prefixsp digit "," space;
		main := (
			newline |
			stream @{ fcall stitle; } |
			duration @{ fcall sduration; } |
			video @{ fcall picture; } |
			autocrop @{ fcall crop; } |
			prefixsp "chapters:" @{ section = CHAPTER; fmt.Printf("chapter"); } |
			prefixsp "audio tracks:" @{ section = AUDIO; fmt.Printf("audio"); } |
			prefixsp "subtitle tracks:" @{ section = SUBTITLE; fmt.Printf("subtitle"); } |
			track @{
				switch section {
				case AUDIO:
					fcall atrack;
				case SUBTITLE:
					fcall subtitle;
				}
			}
		)*;
		write init;
		write exec;
	}%%
	return meta
}

package handbrake

import (
	"fmt"
	"strings"
	"strconv"
)

%%{
	machine handbrake;
	write data;
}%%

type Section int
const (
	NONE Section = iota
	CHAPTER
	AUDIO
	SUBTITLE
)

func parseTime(timestring string) float64 {
	rawTime := strings.Trim(timestring, " \n");
	splitTime := strings.Split(rawTime, ":")
	var length float64
	hours, err := strconv.ParseInt(splitTime[0], 10, 8)
	if err != nil {
		panic(err)
	}
	length += float64(hours * 60 * 60)
	minutes, err := strconv.ParseInt(splitTime[1], 10, 8)
	length += float64(minutes * 60)
	seconds, err := strconv.ParseInt(splitTime[2], 10, 8)
	length += float64(seconds)
	return length
}

func parseOutput(data string) HandBrakeMeta {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	var stack = []int{0}
	var section = NONE
	var meta = HandBrakeMeta{}
	line := 1
	fmt.Printf("%02d: ", line)
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		stitle := |*
			([^.])+[.]alnum+ => {
				meta.Title = strings.Trim(data[ts:te], " \n");
				fmt.Printf("%s", meta.Title);
				fret;
			};
		*|;
		sduration := |*
			space+;
			digit{2}[:]digit{2}[:]digit{2} => {
				meta.Duration = parseTime(data[ts:te])
				fmt.Printf("%f", meta.Duration);
				fret;
			};
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} => { fmt.Printf("%s:", data[ts:te]); fret;};
		*|;
		paspect := |*
			space*;
			digit{1,4} "/" digit{1,4} => { fmt.Printf("%s:", data[ts:te]); fret; };
		*|;
		daspect := |*
			space*;
			digit . "." . digit{1,3} => { fmt.Printf("%s:", data[ts:te]); fret; };
		*|;
		sfps := |*
			"\n" => { ts -= 10; fmt.Printf("%s", data[ts:te-5]); p -= 1; fret; };
		*|;
		crop := |*
			space*;
			digit{1,3} "/" digit{1,3} "/" digit{1,3} "/" digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		atrack := |*
# Language
			[A-Z] alpha+ => { fmt.Printf("a-%s:", data[ts:te]); };
			space;
			"(";
# Codec
			"AC3" | "DTS" | "aac" => { fmt.Printf("b-%s:", data[ts:te]); };
			")";
			space;
			"(";
# Channels
			digit . "." . digit => { fmt.Printf("c-%s:", data[ts:te]) };
			space;
			"ch) ";
# Ignore this bit
			"(iso" digit{3} "-" digit ":" space lower{3} "),";
# Hertz
			space;
			digit+ "Hz," => { fmt.Printf("d-%s:", data[ts:te-3]) };
			space;
# Bps
			digit+ "bps" => { fmt.Printf("e-%s", data[ts:te-3]); fret; };
			*|;
		subtype = "Bitmap" | "Text";
		format = "VOBSUB" | "UTF-8";
		subtitle := |*
			[A-Z] alpha+ - subtype - format => { fmt.Printf("a-%s:", data[ts:te]); };
			space;
			"(iso" digit{3} "-" digit ":" space lower{3} ")";
			space;
			"(" subtype ")" => { fmt.Printf("b-%s:", data[ts+1:te-1]); };
			"(" format ")" => { fmt.Printf("c-%s", data[ts+1:te-1]); fret; };
		*|;
		word = [a-z]+;
		prefix = space+ "+";
		prefixsp = prefix space;
		stream = prefixsp "stream:";
		duration = prefixsp "duration:";
		size = prefixsp "size:";
		pixelaspect = prefix any+ "pixel" space+ "aspect:";
		displayaspect = prefix any+ "display" space+ "aspect:";
		fps = prefix any+ "fps";
		autocrop = prefixsp "autocrop:";
		track = prefixsp digit "," space;
		main := (
			newline |
			stream @{ fcall stitle; } |
			duration @{ fcall sduration; } |
			size @{ fcall picture; } |
			pixelaspect @{ fcall paspect; } |
			displayaspect @{ fcall daspect; } |
			fps @{ fcall sfps; } |
			autocrop @{ fcall crop; } |
			prefixsp "chapters:" @{ section = CHAPTER; fmt.Printf("chapter"); } |
			prefixsp "audio tracks:" @{ section = AUDIO; fmt.Printf("audio"); } |
			prefixsp "subtitle tracks:" @{ section = SUBTITLE; fmt.Printf("subtitle"); } |
			track @{
				switch section {
				case AUDIO:
					fcall atrack;
				case SUBTITLE:
					fcall subtitle;
				}
			}
		)*;
		write init;
		write exec;
	}%%
	return meta
}


validator_ragel: reporting x86-32 error offsets as in x86-64

As in x86-64 report instruction start offsets on error. The code initializing
'begin' may seem redundant, but it is required in x86-64 to avoid compile time
warnings, making the lines exactly match here.

BUG=http://code.google.com/p/nativeclient/issues/detail?id=2597
TEST=too boring without a make target, TBD

Review URL: https://chromiumcodereview.appspot.com/9466049

git-svn-id: 721b910a23eff8a86f00c8fd261a7587cddf18f8@7890 fcba33aa-ac0c-11dd-b9e7-8d5594d729c2

/*
 * Copyright (c) 2011 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <elf.h>
#include <inttypes.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "validator.h"

#undef TRUE
#define TRUE    1

#undef FALSE
#define FALSE   0

#define check_jump_dest \
    if ((jump_dest & bundle_mask) != bundle_mask) { \
      if (jump_dest >= size) { \
        printf("direct jump out of range: %zx\n", jump_dest); \
        result = 1; \
        goto error_detected; \
      } else { \
        BitmapSetBit(jump_dests, jump_dest + 1); \
      } \
    }

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action rel8_operand {
    int8_t offset = (uint8_t) (p[0]);
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }
  action rel16_operand {
    assert(FALSE);
  }
  action rel32_operand {
    int32_t offset =
           (uint32_t) (p[-3] + 256U * (p[-2] + 256U * (p[-1] + 256U * (p[0]))));
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }

  # Do nothing when IMM operand is detected for now.  Will be used later for
  # dynamic code modification support.
  action imm2_operand { }
  action imm8_operand { }
  action imm16_operand { }
  action imm32_operand { }
  action imm64_operand { }
  action imm8_second_operand { }
  action imm16_second_operand { }
  action imm32_second_operand { }
  action imm64_second_operand { }

  include decode_x86_32 "validator-x86_32-instruction.rl";

  special_instruction =
    (0x83 0xe0 0xe0 0xff (0xd0|0xe0)  | # naclcall/jmp %acx
     0x83 0xe1 0xe0 0xff (0xd1|0xe1)  | # naclcall %ecx
     0x83 0xe2 0xe0 0xff (0xd2|0xe2)  | # naclcall %edx
     0x83 0xe3 0xe0 0xff (0xd3|0xe3)  | # naclcall %ebx
     0x83 0xe4 0xe0 0xff (0xd4|0xe4)  | # naclcall %esp
     0x83 0xe5 0xe0 0xff (0xd5|0xe5)  | # naclcall %ebp
     0x83 0xe6 0xe0 0xff (0xd6|0xe6)  | # naclcall %esi
     0x83 0xe7 0xe0 0xff (0xd7|0xe7))   # naclcall %edi
    @{ BitmapClearBit(valid_targets, (p - data) - 1);
    };

  main := ((one_instruction | special_instruction) >{
        begin = p;
        BitmapSetBit(valid_targets, p - data);
     })*
     @{
       /* On successful match the instruction start must point to the next byte
        * to be able to report the new offset as the start of instruction
        * causing error.  */
       begin = p + 1;
     }
    $err{
        process_error(begin, userdata);
        result = 1;
        goto error_detected;
    };

}%%

%% write data;

/* Ignore this information for now.  */
#define data16_prefix if (0) result
#define lock_prefix if (0) result
#define repz_prefix if (0) result
#define repnz_prefix if (0) result
#define branch_not_taken if (0) result
#define branch_taken if (0) result
#define vex_prefix3 if (0) result
#define disp if (0) p
#define disp_type if (0) result

enum disp_mode {
  DISPNONE,
  DISP8,
  DISP16,
  DISP32
};

static const int kBitsPerByte = 8;

static inline uint8_t *BitmapAllocate(uint32_t indexes) {
  uint32_t byte_count = (indexes + kBitsPerByte - 1) / kBitsPerByte;
  uint8_t *bitmap = malloc(byte_count);
  if (bitmap != NULL) {
    memset(bitmap, 0, byte_count);
  }
  return bitmap;
}

static inline int BitmapIsBitSet(uint8_t *bitmap, uint32_t index) {
  return (bitmap[index / kBitsPerByte] & (1 << (index % kBitsPerByte))) != 0;
}

static inline void BitmapSetBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] |= 1 << (index % kBitsPerByte);
}

static inline void BitmapClearBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] &= ~(1 << (index % kBitsPerByte));
}

static int CheckJumpTargets(uint8_t *valid_targets, uint8_t *jump_dests,
                            size_t size) {
  size_t i;
  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      printf("bad jump to around %x\n", (unsigned)(i * 32));
      return 1;
    }
  }
  return 0;
}

int ValidateChunkIA32(const uint8_t *data, size_t size,
                      process_error_func process_error, void *userdata) {
  const size_t bundle_size = 32;
  const size_t bundle_mask = bundle_size - 1;

  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
  const uint8_t *begin = p;  /* Start of the instruction being processed.  */

  int result = 0;

  assert(size % bundle_size == 0);

  while (p < data + size) {
    const uint8_t *pe = p + bundle_size;
    const uint8_t *eof = pe;
    int cs;

    %% write init;
    %% write exec;
  }

  if (CheckJumpTargets(valid_targets, jump_dests, size)) {
    return 1;
  }

error_detected:
  return result;
}

/*
 * Copyright (c) 2011 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <elf.h>
#include <inttypes.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "validator.h"

#undef TRUE
#define TRUE    1

#undef FALSE
#define FALSE   0

#define check_jump_dest \
    if ((jump_dest & bundle_mask) != bundle_mask) { \
      if (jump_dest >= size) { \
        printf("direct jump out of range: %zx\n", jump_dest); \
        result = 1; \
        goto error_detected; \
      } else { \
        BitmapSetBit(jump_dests, jump_dest + 1); \
      } \
    }

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action rel8_operand {
    int8_t offset = (uint8_t) (p[0]);
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }
  action rel16_operand {
    assert(FALSE);
  }
  action rel32_operand {
    int32_t offset =
           (uint32_t) (p[-3] + 256U * (p[-2] + 256U * (p[-1] + 256U * (p[0]))));
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }

  # Do nothing when IMM operand is detected for now.  Will be used later for
  # dynamic code modification support.
  action imm2_operand { }
  action imm8_operand { }
  action imm16_operand { }
  action imm32_operand { }
  action imm64_operand { }
  action imm8_second_operand { }
  action imm16_second_operand { }
  action imm32_second_operand { }
  action imm64_second_operand { }

  include decode_x86_32 "validator-x86_32-instruction.rl";

  special_instruction =
    (0x83 0xe0 0xe0 0xff (0xd0|0xe0)  | # naclcall/jmp %acx
     0x83 0xe1 0xe0 0xff (0xd1|0xe1)  | # naclcall %ecx
     0x83 0xe2 0xe0 0xff (0xd2|0xe2)  | # naclcall %edx
     0x83 0xe3 0xe0 0xff (0xd3|0xe3)  | # naclcall %ebx
     0x83 0xe4 0xe0 0xff (0xd4|0xe4)  | # naclcall %esp
     0x83 0xe5 0xe0 0xff (0xd5|0xe5)  | # naclcall %ebp
     0x83 0xe6 0xe0 0xff (0xd6|0xe6)  | # naclcall %esi
     0x83 0xe7 0xe0 0xff (0xd7|0xe7))   # naclcall %edi
    @{ BitmapClearBit(valid_targets, (p - data) - 1);
    };

  main := ((one_instruction | special_instruction) >{
        BitmapSetBit(valid_targets, p - data);
     })*
    $!{ process_error(p, userdata);
        result = 1;
        goto error_detected;
    };

}%%

%% write data;

/* Ignore this information for now.  */
#define data16_prefix if (0) result
#define lock_prefix if (0) result
#define repz_prefix if (0) result
#define repnz_prefix if (0) result
#define branch_not_taken if (0) result
#define branch_taken if (0) result
#define vex_prefix3 if (0) result
#define disp if (0) p
#define disp_type if (0) result

enum disp_mode {
  DISPNONE,
  DISP8,
  DISP16,
  DISP32
};

static const int kBitsPerByte = 8;

static inline uint8_t *BitmapAllocate(uint32_t indexes) {
  uint32_t byte_count = (indexes + kBitsPerByte - 1) / kBitsPerByte;
  uint8_t *bitmap = malloc(byte_count);
  if (bitmap != NULL) {
    memset(bitmap, 0, byte_count);
  }
  return bitmap;
}

static inline int BitmapIsBitSet(uint8_t *bitmap, uint32_t index) {
  return (bitmap[index / kBitsPerByte] & (1 << (index % kBitsPerByte))) != 0;
}

static inline void BitmapSetBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] |= 1 << (index % kBitsPerByte);
}

static inline void BitmapClearBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] &= ~(1 << (index % kBitsPerByte));
}

static int CheckJumpTargets(uint8_t *valid_targets, uint8_t *jump_dests,
                            size_t size) {
  size_t i;
  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      printf("bad jump to around %x\n", (unsigned)(i * 32));
      return 1;
    }
  }
  return 0;
}

int ValidateChunkIA32(const uint8_t *data, size_t size,
                      process_error_func process_error, void *userdata) {
  const size_t bundle_size = 32;
  const size_t bundle_mask = bundle_size - 1;

  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
/*  const uint8_t *begin;*/

  int result = 0;

  assert(size % bundle_size == 0);

  while (p < data + size) {
    const uint8_t *pe = p + bundle_size;
    const uint8_t *eof = pe;
    int cs;

    %% write init;
    %% write exec;
  }

  if (CheckJumpTargets(valid_targets, jump_dests, size)) {
    result = 1;
    goto error_detected;
  }

error_detected:
  return result;
}