Added initial unicode general properties handling

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_lookahead       = '?=';
  group_nlookahead      = '?!';
  group_lookbehind      = '?<=';
  group_nlookbehind     = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive |
                          group_lookahead | group_nlookahead |
                          group_lookbehind | group_nlookbehind |
                          group_named;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'Nd'; self.emit(type, :number_decimal, text, ts, te)
      when 'Nl'; self.emit(type, :number_letter, text, ts, te)
      when 'No'; self.emit(type, :number_other, text, ts, te)

      # Punctuation
      when 'Pc'; self.emit(type, :punct_connector, text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash, text, ts, te)
      when 'Ps'; self.emit(type, :punct_open, text, ts, te)
      when 'Pe'; self.emit(type, :punct_close, text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial, text, ts, te)
      when 'Pf'; self.emit(type, :punct_final, text, ts, te)
      when 'Po'; self.emit(type, :punct_other, text, ts, te)

      # Symbols
      when 'Sm'; self.emit(type, :symbol_math, text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency, text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier, text, ts, te)
      when 'So'; self.emit(type, :symbol_other, text, ts, te)

      # Separators
      when 'Zs'; self.emit(type, :separator_space, text, ts, te)
      when 'Zl'; self.emit(type, :separator_line, text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph, text, ts, te)

      # Codepoints
      when 'Cc'; self.emit(type, :code_control, text, ts, te)
      when 'Cf'; self.emit(type, :code_format, text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate, text, ts, te)
      when 'Co'; self.emit(type, :code_private, text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 2) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Extended Groups
    #   (subexp)            captured group
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    # ------------------------------------------------------------------------
    group_open . group_type? > (grouped, 1) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?=';  self.emit(:group, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:group, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:group, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:group, :nlookbehind,  text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      self.emit(:quantifier, :zero_or_one, data[ts..te-1].pack('c*'), ts, te)
    };

    zero_or_more {
      self.emit(:quantifier, :zero_or_more, data[ts..te-1].pack('c*'), ts, te)
    };

    one_or_more {
      self.emit(:quantifier, :one_or_more, data[ts..te-1].pack('c*'), ts, te)
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_lookahead       = '?=';
  group_nlookahead      = '?!';
  group_lookbehind      = '?<=';
  group_nlookbehind     = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive |
                          group_lookahead | group_nlookahead |
                          group_lookbehind | group_nlookbehind |
                          group_named;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    property_char . '{' . property_name . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 2) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Extended Groups
    #   (subexp)            captured group
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    # ------------------------------------------------------------------------
    group_open . group_type? > (grouped, 1) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?=';  self.emit(:group, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:group, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:group, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:group, :nlookbehind,  text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      self.emit(:quantifier, :zero_or_one, data[ts..te-1].pack('c*'), ts, te)
    };

    zero_or_more {
      self.emit(:quantifier, :zero_or_more, data[ts..te-1].pack('c*'), ts, te)
    };

    one_or_more {
      self.emit(:quantifier, :one_or_more, data[ts..te-1].pack('c*'), ts, te)
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%


[axl_st] added literals to user_code and user_data machines

otherwise, having a literal with unbalanced {} or () leads to incorrect script extaction


add comments: ignore lines starting with optional space followed by #

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  comment = ( ' '* '#' (any - '\n')* '\n' );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  attr_value = ( alpha (alnum | '-' | '_')** );

  attributes =
    (
      space+
      (
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              '(:'
              (
                attr_value** >{ LX_TK_START(TK_ATTR_CAPTURE); }
                             %{ LX_TK_STOP; }
              )
              ')'
            )
            |
            (
              attr_value** >{ LX_TK_START(TK_ATTR_LITERAL); }
                           %{ LX_TK_STOP; }
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      (
        '  ' >{ LX_TK_START(TK_INDENT); }
             %{ LX_TK_STOP; }
      )*
      (
        '<' >{ LX_TK_START(TK_RULE_BEGIN); }
            %{ LX_TK_STOP; }
      )
      (
        '!' >{ LX_TK_START(TK_DIRECT_DESC); }
            %{ LX_TK_STOP; }
      )?
      (
        cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                  %{ LX_TK_STOP; }
      )?
      (
        attr_name >{ LX_TK_START(TK_TAG_NAME); }
                  %{ LX_TK_STOP; }
      )?
      attributes?
      (
        '>' >{ LX_TK_START(TK_RULE_END); }
            %{ LX_TK_STOP; }
      )
      '\n'
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%


Added a couple of places where line number wasn't incremented

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    /*NSLog(@"emitting %ld = %@ on line: %ld", tokenId, val, parserState.currentLine);*/ \
    Parse(parser, tokenId, [TQToken withId:tokenId value:val line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken((tokenId), [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    char        = ualnum | '?' | '!' | '_';
    constant    = '_'* uupper char*;
    identifier  = '_'* ulower char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    lGuillmt    = 0xC2 0xAB; # «
    rGuillmt    = 0xC2 0xBB; # »

    strCont     = (ualnum | ascii) - '"';
    simpleStr   = '"' strCont* '"';
    constStr    = "@" (simpleStr | [^\n ;,\]}.)`]+);
    selector    = (ualnum | "+" | "-" | "*" | "/" | "^" | "=" | "~" | "<" | ">" | "[" "]" | "_")* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\n'                             => { [strBuf appendString:@"\n"]; IncrementLine();                   };
    '\\n'                            => { [strBuf appendString:@"\n"];                                    };
    '\\t'                            => { [strBuf appendString:@"\t"];                                    };
    '\\r'                            => { [strBuf appendString:@"\r"];                                    };
    '\\"'                            => { [strBuf appendString:@"\""];                                    };
    "\\'"                            => { [strBuf appendString:@"'"];                                     };
    "\\\\"                           => { [strBuf appendString:@"\\"];                                    };
    "\\" lGuillmt                    => { [strBuf appendString:@"«"];                                     };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        [strBuf appendFormat:@"%c", (int)code];
    };
    "\\" ualnum+                     => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                 };
    strCont                          => { [strBuf appendString:NSStr(0, 0)];                              };

    lGuillmt                         => { _EmitToken(temp3 == 1 ? LSTR  : MSTR, strBuf); fret;            };
    '"' term                         => { _EmitToken(temp3 == 1 ? STRNL : RSTRNL, strBuf);
                                          BacktrackTerm(); fret;                                          };
    '"'                              => { _EmitToken(temp3 == 1 ? STR   : RSTR, strBuf); fret;            };
*|;

regex := |*
    '\n'                             => { [strBuf appendString:@"\n"]; IncrementLine();                   };
    "\\/"                            => { [strBuf appendString:@"\\/"];                                   };
    "\\"                             => { [strBuf appendString:@"\\"];                                    };
    regexCont                        => { [strBuf appendString:NSStr(0, 0)];                              };
    "/" [im]* term                   => { [strBuf appendString:@"/"]; _EmitToken(REGEXNL, strBuf);
                                          BacktrackTerm(); fret;                                          };
    "/" [im]*                        => { [strBuf appendString:@"/"]; _EmitToken(REGEX, strBuf);   fret;  };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            strBuf = [NSMutableString stringWithString:@"/"];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "#"                              => { EmitToken(HASH);                                                };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { temp3 = 1; strBuf = [NSMutableString string]; fcall string;     };
    rGuillmt                         => { temp3 = 2; strBuf = [NSMutableString string]; fcall string;     };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableString *strBuf;

    unsigned char *temp1, *temp2;
    int temp3 = 0;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    /*NSLog(@"emitting %ld = %@ on line: %ld", tokenId, val, parserState.currentLine);*/ \
    Parse(parser, tokenId, [TQToken withId:tokenId value:val line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken((tokenId), [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    char        = ualnum | '?' | '!' | '_';
    constant    = '_'* uupper char*;
    identifier  = '_'* ulower char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    lGuillmt    = 0xC2 0xAB; # «
    rGuillmt    = 0xC2 0xBB; # »

    strCont     = (ualnum | ascii) - '"';
    simpleStr   = '"' strCont* '"';
    constStr    = "@" (simpleStr | [^\n ;,\]}.)`]+);
    selector    = (ualnum | "+" | "-" | "*" | "/" | "^" | "=" | "~" | "<" | ">" | "[" "]" | "_")* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\\n'                            => { [strBuf appendString:@"\n"];                                    };
    '\\t'                            => { [strBuf appendString:@"\t"];                                    };
    '\\r'                            => { [strBuf appendString:@"\r"];                                    };
    '\\"'                            => { [strBuf appendString:@"\""];                                    };
    "\\'"                            => { [strBuf appendString:@"'"];                                     };
    "\\\\"                           => { [strBuf appendString:@"\\"];                                    };
    "\\" lGuillmt                    => { [strBuf appendString:@"«"];                                     };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        [strBuf appendFormat:@"%c", (int)code];
    };
    "\\" ualnum+                     => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                 };
    strCont                          => { [strBuf appendString:NSStr(0, 0)];                              };

    lGuillmt                         => { _EmitToken(temp3 == 1 ? LSTR  : MSTR, strBuf); fret;            };
    '"' term                         => { _EmitToken(temp3 == 1 ? STRNL : RSTRNL, strBuf);
                                          BacktrackTerm(); fret;                                          };
    '"'                              => { _EmitToken(temp3 == 1 ? STR   : RSTR, strBuf); fret;            };
*|;

regex := |*
    "\\/"                            => { [strBuf appendString:@"\\/"];                                   };
    "\\"                             => { [strBuf appendString:@"\\"];                                    };
    regexCont                        => { [strBuf appendString:NSStr(0, 0)];                              };
    "/" [im]* term                   => { [strBuf appendString:@"/"]; _EmitToken(REGEXNL, strBuf);
                                          BacktrackTerm(); fret;                                          };
    "/" [im]*                        => { [strBuf appendString:@"/"]; _EmitToken(REGEX, strBuf);   fret;  };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            strBuf = [NSMutableString stringWithString:@"/"];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
#    "/="                             => { EmitToken(ASSIGNDIV);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "#"                              => { EmitToken(HASH);                                                };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { temp3 = 1; strBuf = [NSMutableString string]; fcall string;     };
    rGuillmt                         => { temp3 = 2; strBuf = [NSMutableString string]; fcall string;     };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableString *strBuf;

    unsigned char *temp1, *temp2;
    int temp3 = 0;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}


[jnc_ct] lexer should accept unterminated c-style comments


Small cleanup for the XPath lexer.

Small cleanup for the XPath lexer.

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Lexer for turning XPath expressions into a set of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a symbol
    # 2. The token value or nil if there is no value
    #
    # Basic usage of this lexer is as following:
    #
    #     lexer  = Oga::XPath::Lexer.new('//foo/bar')
    #     tokens = lexer.lex
    #
    # Alternatively you can stream tokens instead of returning them as a whole:
    #
    #     lexer = Oga::XPath::Lexer.new('//foo/bar')
    #
    #     lexer.advance do |type, value|
    #
    #     end
    #
    # Unlike the XML lexer the XPath lexer does not support IO instances, it can
    # only lex strings.
    #
    # ## Thread Safety
    #
    # This class keeps track of an internal state. As a result it's not safe to
    # share a single instance between multiple threads. However, you're free to
    # use separate instances per thread as there is no global (= class level)
    # shared state.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # Axes that require a separate `node()` call to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_NODE = %w{descendant-or-self parent self}

      ##
      # Axes that require an extra T_SLASH token to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_EXTRA_SLASH = %w{descendant-or-self}

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };
        lbrack = '[' @{ add_token(:T_LBRACK) };
        rbrack = ']' @{ add_token(:T_RBRACK) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = [a-zA-Z*]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = ('-' | '+')* digit+;
        float   = ('-' | '+')* digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)

          # Short axes that use node() as their default, implicit test. This is
          # added on lexer level to make it easier to handle these cases on
          # parser/evaluator level.
          if AXIS_EMIT_NODE.include?(value)
            add_token(:T_TYPE_TEST, 'node')

            if AXIS_EMIT_EXTRA_SLASH.include?(value) and te != eof
              add_token(:T_SLASH)
            end
          end
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        op_pipe = '|'  %{ add_token(:T_PIPE) };
        op_plus = '+'  %{ add_token(:T_ADD) };
        op_eq   = '='  %{ add_token(:T_EQ) };
        op_neq  = '!=' %{ add_token(:T_NEQ) };
        op_lt   = '<'  %{ add_token(:T_LT) };
        op_gt   = '>'  %{ add_token(:T_GT) };
        op_lte  = '<=' %{ add_token(:T_LTE) };
        op_gte  = '>=' %{ add_token(:T_GTE) };

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.

        op_and = ' and ' %{ add_token(:T_AND) };
        op_or  = ' or '  %{ add_token(:T_OR) };
        op_div = ' div ' %{ add_token(:T_DIV) };
        op_mod = ' mod ' %{ add_token(:T_MOD) };
        op_mul = ' * '   %{ add_token(:T_MUL) };
        op_sub = ' - '   %{ add_token(:T_SUB) };

        operator = op_pipe
          | op_and
          | op_or
          | op_plus
          | op_div
          | op_mod
          | op_eq
          | op_neq
          | op_lt
          | op_gt
          | op_lte
          | op_gte
          | op_mul
          | op_sub
          ;

        # Node type tests
        #
        # While these look like functions they are actually node type tests. For
        # example, comment() matches all comment nodes.
        #
        # See http://www.w3.org/TR/xpath/#NT-NodeType for more information.

        type_test = (
          'comment' |
          'text' |
          'processing-instruction' |
          'node'
        ) '()';

        action emit_type_test {
          emit(:T_TYPE_TEST, ts, te - 2)
        }

        # Variables
        #
        # XPath 1.0 allows the use of variables in expressions. Oddly enough you
        # can not assign variables in an expression, you can only refer to them.
        # This means that libraries themselves have to expose an interface for
        # setting variables.
        #
        var = '$' identifier;

        action emit_variable {
          emit(:T_VAR, ts + 1, te)
        }

        main := |*
          operator;
          whitespace | slash | lparen | rparen | comma | colon | lbrack | rbrack;

          type_test  => emit_type_test;
          var        => emit_variable;
          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Lexer for turning XPath expressions into a set of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a symbol
    # 2. The token value or nil if there is no value
    #
    # Basic usage of this lexer is as following:
    #
    #     lexer  = Oga::XPath::Lexer.new('//foo/bar')
    #     tokens = lexer.lex
    #
    # Alternatively you can stream tokens instead of returning them as a whole:
    #
    #     lexer = Oga::XPath::Lexer.new('//foo/bar')
    #
    #     lexer.advance do |type, value|
    #
    #     end
    #
    # Unlike the XML lexer the XPath lexer does not support IO instances, it can
    # only lex strings.
    #
    # ## Thread Safety
    #
    # This class keeps track of an internal state. As a result it's not safe to
    # share a single instance between multiple threads. However, you're free to
    # use separate instances per thread as there is no global (= class level)
    # shared state.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # Axes that require a separate `node()` call to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_NODE = %w{descendant-or-self parent self}

      ##
      # Axes that require an extra T_SLASH token to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_EXTRA_SLASH = %w{descendant-or-self}

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = [a-zA-Z*]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = ('-' | '+')* digit+;
        float   = ('-' | '+')* digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)

          # Short axes that use node() as their default, implicit test. This is
          # added on lexer level to make it easier to handle these cases on
          # parser/evaluator level.
          if AXIS_EMIT_NODE.include?(value)
            add_token(:T_TYPE_TEST, 'node')

            if AXIS_EMIT_EXTRA_SLASH.include?(value) and te != eof
              add_token(:T_SLASH)
            end
          end
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        op_pipe = '|'  %{ add_token(:T_PIPE) };
        op_plus = '+'  %{ add_token(:T_ADD) };
        op_eq   = '='  %{ add_token(:T_EQ) };
        op_neq  = '!=' %{ add_token(:T_NEQ) };
        op_lt   = '<'  %{ add_token(:T_LT) };
        op_gt   = '>'  %{ add_token(:T_GT) };
        op_lte  = '<=' %{ add_token(:T_LTE) };
        op_gte  = '>=' %{ add_token(:T_GTE) };

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.

        op_and = ' and ' %{ add_token(:T_AND) };
        op_or  = ' or '  %{ add_token(:T_OR) };
        op_div = ' div ' %{ add_token(:T_DIV) };
        op_mod = ' mod ' %{ add_token(:T_MOD) };
        op_mul = ' * '   %{ add_token(:T_MUL) };
        op_sub = ' - '   %{ add_token(:T_SUB) };

        operator = op_pipe
          | op_and
          | op_or
          | op_plus
          | op_div
          | op_mod
          | op_eq
          | op_neq
          | op_lt
          | op_gt
          | op_lte
          | op_gte
          | op_mul
          | op_sub
          ;

        # Node type tests
        #
        # While these look like functions they are actually node type tests. For
        # example, comment() matches all comment nodes.
        #
        # See http://www.w3.org/TR/xpath/#NT-NodeType for more information.

        type_test = 'comment' | 'text' | 'processing-instruction' | 'node';

        action emit_type_test {
          emit(:T_TYPE_TEST, ts, te - 2)
        }

        # Variables
        #
        # XPath 1.0 allows the use of variables in expressions. Oddly enough you
        # can not assign variables in an expression, you can only refer to them.
        # This means that libraries themselves have to expose an interface for
        # setting variables.
        #
        var = '$' identifier;

        action emit_variable {
          emit(:T_VAR, ts + 1, te)
        }

        main := |*
          operator;
          whitespace | slash | lparen | rparen | comma | colon;

          type_test '()' => emit_type_test;

          '[' => { add_token(:T_LBRACK) };
          ']' => { add_token(:T_RBRACK) };

          var        => emit_variable;
          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga


Add retrieval and storage of parser state

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    /*mark_key = MARK_STR_NEW();*/
    /*mark = NULL;*/
  }

  action write_header {
    /*rb_funcall(mark_message, g_write_header, 2, mark_key, MARK_STR_NEW());*/
    /*mark_key = Qnil;*/
    /*mark = NULL;*/
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    /*rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());*/
    /*mark = NULL;*/
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private class State {
    public int cs = JavaParser.start;
    public byte[] chunk;
    public int mark = -1;
    public RubyString mark_key;
    public IRubyObject mark_message;
    public int mark_message_size = -1;
    public int mark_content_length = -1;
  }

  private RaiseException exception;
  private long maxMessageSize;
  private State state;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
    state = new State();
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();

    int p = 0;
    int pe = data.length;

    int cs = state.cs;
    int mark = state.mark;
    RubyString mark_key = state.mark_key;
    IRubyObject mark_message = state.mark_message;
    int mark_message_size = state.mark_message_size;
    int mark_content_length = state.mark_content_length;

    %% write exec;

    state.cs = cs;
    state.mark = mark;
    state.mark_key = mark_key;
    state.mark_message = mark_message;
    state.mark_message_size = mark_message_size;
    state.mark_content_length = mark_content_length;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    /*mark_key = MARK_STR_NEW();*/
    /*mark = NULL;*/
  }

  action write_header {
    /*rb_funcall(mark_message, g_write_header, 2, mark_key, MARK_STR_NEW());*/
    /*mark_key = Qnil;*/
    /*mark = NULL;*/
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    /*rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());*/
    /*mark = NULL;*/
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private long maxMessageSize;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();
    int p = 0;
    int pe = data.length;
    int cs = start;

    IRubyObject mark_message = null;
    int mark = -1;

    %% write exec;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}