Extracted Unicode properties scanner into a separate machine for reuse in other machines

%%{
  machine re_scanner;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  # Property names are being treated as case-insensitive, but it is not clear
  # yet if this applies to all flavors and in all encodings. A bug has just
  # been filed against ruby regarding this issue.
  # http://redmine.ruby-lang.org/issues/show/4014
  property_char         = [pP];

  property_name_unicode = 'alnum'i | 'alpha'i | 'any'i   | 'ascii'i | 'blank'i |
                          'cntrl'i | 'digit'i | 'graph'i | 'lower'i | 'print'i |
                          'punct'i | 'space'i | 'upper'i | 'word'i  | 'xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  property_sequence     = property_char.'{'.(property_name | general_category).'}';

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha; # FIXME: incorrect, and can include escapes

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifier_range_bre  = backslash . range_open . (digit+)? . ','? . (digit+)? .
                          backslash . range_close;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  # EOF error, used where it can be detected
  action premature_end_error { raise "Premature end of pattern" }

  # group (nesting) and set open/close actions
  action group_opened { in_group += 1 }
  action group_closed { in_group -= 1 }

  action set_opened { in_set = true }
  action set_closed { in_set = false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix @err(premature_end_error) {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    meta_char {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts-1, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    char_type {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    (ascii_print - char_type) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # FIXME: a backspace only inside a set
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    property_char > (escaped_alpha, 2) {
      fhold; fcall unicode_property; fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Unicode properties scanner
  # --------------------------------------------------------------------------
  unicode_property := |*
    property_sequence < err(premature_end_error) {
      text = data[ts-1..te-1].pack('c*')

      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open %set_opened  {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, all transitions
    group_open . group_comment $group_closed {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options >group_opened {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open @group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    quantifier_range  @err(premature_end_error) {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    quantifier_range_bre  @err(premature_end_error) {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returns at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group = 0
    in_set   = false

    %% write init;
    %% write exec;

    raise "Premature end of pattern (missing group closing paranthesis) [#{in_group}]" if
      in_group > 0
    raise "Premature end of pattern (missing set closing bracket)" if in_set

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join
    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  # Property names are being treated as case-insensitive, but it is not clear
  # yet if this applies to all flavors and in all encodings. A bug has just
  # been filed against ruby regarding this issue.
  # http://redmine.ruby-lang.org/issues/show/4014
  property_char         = [pP];
  property_name_unicode = 'alnum'i | 'alpha'i | 'any'i   | 'ascii'i | 'blank'i |
                          'cntrl'i | 'digit'i | 'graph'i | 'lower'i | 'print'i |
                          'punct'i | 'space'i | 'upper'i | 'word'i  | 'xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  property_sequence     = property_char.'{'.(property_name | general_category).'}';

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha; # FIXME: incorrect, and can include escapes

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifier_range_bre  = backslash . range_open . (digit+)? . ','? . (digit+)? .
                          backslash . range_close;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  # EOF error, used where it can be detected
  action premature_end_error { raise "Premature end of pattern" }

  # group (nesting) and set open/close actions
  action group_opened { in_group += 1 }
  action group_closed { in_group -= 1 }

  action set_opened { in_set = true }
  action set_closed { in_set = false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix @err(premature_end_error) {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    meta_char {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts-1, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    char_type {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    (ascii_print - char_type) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # FIXME: a backspace only inside a set
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_sequence  >(escaped_alpha, 2) @err(premature_end_error) {
      text = data[ts-1..te-1].pack('c*')

      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
      fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open %set_opened  {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, all transitions
    group_open . group_comment $group_closed {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options >group_opened {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open @group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    quantifier_range  @err(premature_end_error) {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    quantifier_range_bre  @err(premature_end_error) {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returns at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group = 0
    in_set   = false

    %% write init;
    %% write exec;

    raise "Premature end of pattern (missing group closing paranthesis) [#{in_group}]" if
      in_group > 0
    raise "Premature end of pattern (missing set closing bracket)" if in_set

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join
    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner


[Fix] Fix stack growing

[Fix] Fix stack growing

%%{
  machine smtp_received;

  include smtp_whitespace "smtp_whitespace.rl";
  include smtp_ip "smtp_ip.rl";
  include smtp_date "smtp_date.rl";
  include smtp_address"smtp_address.rl";

  # http://tools.ietf.org/html/rfc5321#section-4.4

  Addtl_Link     = Atom;
  Link           = "TCP" | Addtl_Link;
  Attdl_Protocol = Atom;
  Protocol       = "ESMTP" %ESMTP_proto |
                   "SMTP" %SMTP_proto |
                   "ESMTPS" %ESMTPS_proto |
                   "ESMTPA" %ESMTPA_proto |
                   "LMTP" %LMTP_proto |
                   "IMAP" %IMAP_proto |
                   Attdl_Protocol;

  TCP_info       = address_literal >Real_IP_Start %Real_IP_End |
                  ( Domain >Real_Domain_Start %Real_Domain_End FWS address_literal >Real_IP_Start %Real_IP_End );
  Extended_Domain  = Domain >Real_Domain_Start %Real_Domain_End | # Used to be a real domain
                  ( Domain >Reported_Domain_Start %Reported_Domain_End FWS "(" TCP_info ")" ) | # Here domain is something specified by remote side
                  ( address_literal >Real_Domain_Start %Real_Domain_End FWS "(" TCP_info ")" ) |
                  address_literal >Real_IP_Start %Real_IP_End; # Not RFC conforming, but many MTA try this

  ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; };
  balanced_ccontent := ccontent* ')' @{ fret; };
  comment        =   "(" (FWS? ccontent)* FWS? ")";
  CFWS           =   ((FWS? comment)+ FWS?) | FWS;

  From_domain    = "FROM"i FWS Extended_Domain >From_Start %From_End;
  By_domain      = "BY"i FWS Extended_Domain >By_Start %By_End;

  Via            = CFWS "VIA"i FWS Link;
  With           = CFWS "WITH"i FWS Protocol;

  id_left        = dot_atom_text;
  no_fold_literal = "[" dtext* "]";
  id_right       = dot_atom_text | no_fold_literal;
  msg_id         = "<" id_left "@" id_right ">";
  ID             = CFWS "ID"i FWS ( Atom | msg_id );

  For            = CFWS "FOR"i FWS ( Path | Mailbox ) %For_End;
  Additional_Registered_Clauses  = CFWS Atom FWS String;
  Opt_info       = Via? With? ID? For? Additional_Registered_Clauses?;
  # Here we make From part optional just because many received headers lack it
  Received       = From_domain? CFWS? By_domain? CFWS? Opt_info CFWS? ";" FWS date_time >Date_Start %Date_End CFWS?;

  prepush {
    if (top >= st_storage.size) {
      st_storage.size = (top + 1) * 2;
      st_storage.data = realloc (st_storage.data, st_storage.size * sizeof (int));
      g_assert (st_storage.data != NULL);
      stack = st_storage.data;
    }
  }
}%%

%%{
  machine smtp_received;

  include smtp_whitespace "smtp_whitespace.rl";
  include smtp_ip "smtp_ip.rl";
  include smtp_date "smtp_date.rl";
  include smtp_address"smtp_address.rl";

  # http://tools.ietf.org/html/rfc5321#section-4.4

  Addtl_Link     = Atom;
  Link           = "TCP" | Addtl_Link;
  Attdl_Protocol = Atom;
  Protocol       = "ESMTP" %ESMTP_proto |
                   "SMTP" %SMTP_proto |
                   "ESMTPS" %ESMTPS_proto |
                   "ESMTPA" %ESMTPA_proto |
                   "LMTP" %LMTP_proto |
                   "IMAP" %IMAP_proto |
                   Attdl_Protocol;

  TCP_info       = address_literal >Real_IP_Start %Real_IP_End |
                  ( Domain >Real_Domain_Start %Real_Domain_End FWS address_literal >Real_IP_Start %Real_IP_End );
  Extended_Domain  = Domain >Real_Domain_Start %Real_Domain_End | # Used to be a real domain
                  ( Domain >Reported_Domain_Start %Reported_Domain_End FWS "(" TCP_info ")" ) | # Here domain is something specified by remote side
                  ( address_literal >Real_Domain_Start %Real_Domain_End FWS "(" TCP_info ")" ) |
                  address_literal >Real_IP_Start %Real_IP_End; # Not RFC conforming, but many MTA try this

  ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; };
  balanced_ccontent := ccontent* ')' @{ fret; };
  comment        =   "(" (FWS? ccontent)* FWS? ")";
  CFWS           =   ((FWS? comment)+ FWS?) | FWS;

  From_domain    = "FROM"i FWS Extended_Domain >From_Start %From_End;
  By_domain      = "BY"i FWS Extended_Domain >By_Start %By_End;

  Via            = CFWS "VIA"i FWS Link;
  With           = CFWS "WITH"i FWS Protocol;

  id_left        = dot_atom_text;
  no_fold_literal = "[" dtext* "]";
  id_right       = dot_atom_text | no_fold_literal;
  msg_id         = "<" id_left "@" id_right ">";
  ID             = CFWS "ID"i FWS ( Atom | msg_id );

  For            = CFWS "FOR"i FWS ( Path | Mailbox ) %For_End;
  Additional_Registered_Clauses  = CFWS Atom FWS String;
  Opt_info       = Via? With? ID? For? Additional_Registered_Clauses?;
  # Here we make From part optional just because many received headers lack it
  Received       = From_domain? CFWS? By_domain? CFWS? Opt_info CFWS? ";" FWS date_time >Date_Start %Date_End CFWS?;

  prepush {
    if (top >= st_storage.size) {
      st_storage.data = realloc (st_storage.data, (top + 1) * 2);
      g_assert (st_storage.data != NULL);
      stack = st_storage.data;
    }
  }
}%%


pop statement stack to reach the first statement

=begin
%%{
  machine bel;

  action call_statement {fcall statement;}
  action statement {
    @statement = @statement_stack.pop
    @statement.annotations = @annotations.clone()

    if @statement_group
      statement_group.statements << @statement
    end

    changed
    notify_observers(@statement)
  }
  action statement_init {
    @statement = BEL::Script::Statement.new()
    @statement_stack = [@statement]
  }
  action statement_subject {
    @statement_stack.last.subject = @term

    changed
    notify_observers(@term)
  }
  action statement_oterm {
    @statement_stack.last.object = @term

    changed
    notify_observers(@term)
  }
  action statement_ostmt {
    nested = BEL::Script::Statement.new()
    @statement_stack.last.object = nested
    @statement_stack.push nested
  }
  action statement_pop {
    @statement = @statement_stack.pop
  }
  action rels {@relbuffer = []}
  action reln {@relbuffer << fc}
  action rele {
    rel = @relbuffer.map(&:chr).join()
    @statement_stack.last.rel = rel
  }

  include 'common.rl';
  include 'set.rl';
  include 'term.rl';

  RELATIONSHIP = ('increases'|'->'|'decreases'|'-|'|'directlyIncreases'|'=>'|
                  'directlyDecreases'|'=|'|'causesNoChange'|
                  'positiveCorrelation'|'negativeCorrelation'|
                  'translatedTo'|'>>'|'transcribedTo'|':>'|'isA'|
                  'subProcessOf'|'rateLimitingStepOf'|'biomarkerFor'|
                  'prognosticBiomarkerFor'|'orthologous'|'analogous'|
                  'association'|'--'|'hasMembers'|'hasComponents'|
                  'hasMember'|'hasComponent');

  statement :=
    FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @term_init @call_term SP* %statement_subject '\n'? @statement @return
    RELATIONSHIP >rels $reln %rele SP+
    (
      FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @term_init @call_term %statement_oterm SP* ')'? @return
      |
      '(' @statement_ostmt @call_statement %statement_pop
    ) '\n' @statement @return;

  statement_main :=
    (
      '\n' |
      FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @statement_init @call_statement
    )+;
}%%
=end

module BEL
  Parameter = Struct.new(:ns, :value)
  Term = Struct.new(:fx, :args) do
    def <<(item)
      self.args << item
    end
  end
  Statement = Struct.new(:subject, :rel, :object, :annotations, :comment) do
      def subject_only?
        !rel
      end
      def simple?
        object.is_a? TermDefinition
      end
      def nested?
        object.is_a? StatementDefinition
      end
  end
end

class Parser

  def initialize
    @items = []
    %% write data;
  end

  def exec(input)
    buffer = []
    stack = []
    data = input.read.unpack('c*')

    %% write init;
    %% write exec;
  end
end
Parser.new.exec(ARGV[0] ? File.open(ARGV[0]) : $stdin)

=begin
%%{
  machine bel;

  action call_statement {fcall statement;}
  action statement {
    @statement.annotations = @annotations.clone()

    if @statement_group
      statement_group.statements << @statement
    end

    changed
    notify_observers(@statement)
  }
  action statement_init {
    @statement = BEL::Script::Statement.new()
    @statement_stack = [@statement]
  }
  action statement_subject {
    @statement_stack.last.subject = @term

    changed
    notify_observers(@term)
  }
  action statement_oterm {
    @statement_stack.last.object = @term

    changed
    notify_observers(@term)
  }
  action statement_ostmt {
    nested = BEL::Script::Statement.new()
    @statement_stack.last.object = nested
    @statement_stack.push nested
  }
  action statement_pop {
    @statement = @statement_stack.pop
  }
  action rels {@relbuffer = []}
  action reln {@relbuffer << fc}
  action rele {
    rel = @relbuffer.map(&:chr).join()
    @statement_stack.last.rel = rel
  }

  include 'common.rl';
  include 'set.rl';
  include 'term.rl';

  RELATIONSHIP = ('increases'|'->'|'decreases'|'-|'|'directlyIncreases'|'=>'|
                  'directlyDecreases'|'=|'|'causesNoChange'|
                  'positiveCorrelation'|'negativeCorrelation'|
                  'translatedTo'|'>>'|'transcribedTo'|':>'|'isA'|
                  'subProcessOf'|'rateLimitingStepOf'|'biomarkerFor'|
                  'prognosticBiomarkerFor'|'orthologous'|'analogous'|
                  'association'|'--'|'hasMembers'|'hasComponents'|
                  'hasMember'|'hasComponent');

  statement :=
    FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @term_init @call_term SP* %statement_subject '\n'? @statement @return
    RELATIONSHIP >rels $reln %rele SP+
    (
      FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @term_init @call_term %statement_oterm SP* ')'? @return
      |
      '(' @statement_ostmt @call_statement %statement_pop
    ) '\n' @statement @return;

  statement_main :=
    (
      '\n' |
      FUNCTION >{n = 0} ${n += 1} @{fpc -= n} @statement_init @call_statement
    )+;
}%%
=end

module BEL
  Parameter = Struct.new(:ns, :value)
  Term = Struct.new(:fx, :args) do
    def <<(item)
      self.args << item
    end
  end
  Statement = Struct.new(:subject, :rel, :object, :annotations, :comment) do
      def subject_only?
        !rel
      end
      def simple?
        object.is_a? TermDefinition
      end
      def nested?
        object.is_a? StatementDefinition
      end
  end
end

class Parser

  def initialize
    @items = []
    %% write data;
  end

  def exec(input)
    buffer = []
    stack = []
    data = input.read.unpack('c*')

    %% write init;
    %% write exec;
  end
end
Parser.new.exec(ARGV[0] ? File.open(ARGV[0]) : $stdin)


Cleanup

Cleanup

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include "parse.h"
#import <Tranquil/CodeGen/TQNodeBlock.h>

typedef struct {
    int currentLine;
    TQNodeRootBlock *root;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    Parse(parser, tokenId, val, &parserState); \
} while(0);

#define EmitToken(tokenId) EmitStringToken(tokenId, 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken((tokenId), [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;

    char        = ualpha | '?' | '!';
    constant    = uupper char*;
    identifier  = ulower char*;
    comment     = '\\' [^\n]*;
    newline     = (comment? '\n' space?)+;
    whitespace  = (newline|space|comment);

    lGuillmt    = 0xC2 0xAB; # «
    rGuillmt    = 0xC2 0xBB; # »
    strCont     = '\\' any | (ascii - '"') | ualpha;
    string      = '"' strCont* '"';
    lStr        = '"' strCont* lGuillmt;
    mStr        = rGuillmt strCont* lGuillmt;
    rStr        = rGuillmt strCont* '"';
    selector    = (constant | identifier)? ':';

main := |*
# Hacks
    "{" whitespace* %{ temp1 = p; } identifier %{ temp2 = p; } whitespace* "=" [^>] %{ EmitStringToken(LBRACEDEFARG, 0, 0);   };
    "`" whitespace* %{ temp1 = p; } identifier %{ temp2 = p; } whitespace* "=" [^>] %{ EmitStringToken(BACKTICKDEFARG, 0, 0); };

# Symbols
    "{"                              => { EmitToken(LBRACE);                            };
    "}" newline                      => { EmitToken(RBRACENL);                          };
    "}"                              => { EmitToken(RBRACE);                            };
    "["                              => { EmitToken(LBRACKET);                          };
    "]" newline                      => { EmitToken(RBRACKETNL);                        };
    "]"                              => { EmitToken(RBRACKET);                          };
    "("                              => { EmitToken(LPAREN);                            };
    ")" newline                      => { EmitToken(RPARENNL);                          };
    ")"                              => { EmitToken(RPAREN);                            };
    ","                              => { EmitToken(COMMA);                             };
    "`" newline                      => { EmitToken(BACKTICKNL);                        };
    "`"                              => { EmitToken(BACKTICK);                          };
    ";"                              => { };
    "="                              => { EmitToken(ASSIGN);                            };
    "+"                              => { EmitToken(PLUS);                              };
    "-"                              => { EmitToken(MINUS);                             };
    "--"                             => { EmitToken(DECR);                              };
    "--" newline                     => { EmitToken(DECRNL);                            };
    "++"                             => { EmitToken(INCR);                              };
    "++" newline                     => { EmitToken(INCRNL);                            };
    "*"                              => { EmitToken(ASTERISK);                          };
    "/"                              => { EmitToken(FSLASH);                            };
    "~"                              => { EmitToken(TILDE);                             };
    "#"                              => { EmitToken(HASH);                              };
    "|"                              => { EmitToken(PIPE);                              };
    "^"                              => { EmitToken(CARET);                             };
    "="                              => { EmitToken(ASSIGN);                            };
    "=="                             => { EmitToken(EQUAL);                             };
    "~="                             => { EmitToken(INEQUAL);                           };
    "<"                              => { EmitToken(LESSER);                            };
    ">"                              => { EmitToken(GREATER);                           };
    "<="                             => { EmitToken(LEQUAL);                            };
    ">="                             => { EmitToken(GEQUAL);                            };
    "<<"                             => { };
    ">>"                             => { };
    "=>"                             => { EmitToken(DICTSEP);                           };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1);               };

# Keywords
#    "if"                             => { EmitStringToken(IF);                          };
#    "else"                           => { EmitStringToken(ELSE);                        };
    "and"|"||"                       => { EmitToken(AND);                               };
    "or"|"&&"                        => { EmitToken(OR);                                };
#    "while"                          => { EmitToken(WHILE);                             };
#    "until"                          => { EmitToken(UNTIL);                             };
#    "import"                         => { EmitToken(IMPORT);                            };
#    "async"                          => { EmitToken(ASYNC);                             };
#    "break"                          => { EmitToken(BREAK);                             };
#    "skip"                           => { EmitToken(SKIP);                              };
    # These are identifier keywords used as expressions => need a *NL variant as well
    "self"    newline                => { EmitStringToken(SELFNL,    0, 0);             };
    "super"   newline                => { EmitStringToken(SUPERNL,   0, 0);             };
    "..."     newline                => { EmitStringToken(VAARGNL,   0, 0);             };
    "nil"     newline                => { EmitStringToken(NILNL,     0, 0);             };
    "valid"   newline                => { EmitStringToken(VALIDNL,   0, 0);             };
    "yes"     newline                => { EmitStringToken(YESNL,     0, 0);             };
    "no"      newline                => { EmitStringToken(NONL,      0, 0);             };
    "nothing" newline                => { EmitStringToken(NOTHINGNL, 0, 0);             };
    "self"                           => { EmitStringToken(SELF,      0, 0);             };
    "super"                          => { EmitStringToken(SUPER,     0, 0);             };
    "..."                            => { EmitStringToken(VAARG,     0, 0);             };
    "nil"                            => { EmitStringToken(NIL,       0, 0);             };
    "valid"                          => { EmitStringToken(VALID,     0, 0);             };
    "yes"                            => { EmitStringToken(YES,       0, 0);             };
    "no"                             => { EmitStringToken(NO,        0, 0);             };
    "nothing"                        => { EmitStringToken(NOTHING,   0, 0);             };

# Identifiers
    identifier %{temp1 = p;} newline => { te = temp1; EmitStringToken(IDENTNL, 0, 1);   };
    identifier                       => { EmitStringToken(IDENT,   0, 0);               };

    constant   %{temp1 = p;} newline => { te = temp1; EmitStringToken(CONSTNL, 0, 1);   };
    constant                         => { EmitStringToken(CONST,   0, 0);               };

# Literals
    int   newline                    => { EmitIntToken(NUMBERNL, 10, 0);                };
    float newline                    => { EmitFloatToken(NUMBERNL);                     };
    bin   newline                    => { EmitIntToken(NUMBERNL, 2,  2);                };
    oct   newline                    => { EmitIntToken(NUMBERNL, 8,  2);                };
    hex   newline                    => { EmitIntToken(NUMBERNL, 16, 2);                };

    int                              => { EmitIntToken(NUMBER, 10, 0);                  };
    float                            => { EmitFloatToken(NUMBER);                       };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                  };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                  };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                  };

    lStr                             => { EmitStringToken(LSTR,   1, 2);                };
    mStr                             => { EmitStringToken(MSTR,   2, 2);                };
    rStr                             => { EmitStringToken(RSTR,   2, 1);                };
    rStr   %{temp1 = p;} newline     => { te = temp1; EmitStringToken(RSTRNL, 2, 1);    };
    string                           => { EmitStringToken(STR,    1, 2);                };
    string %{temp1 = p;} newline     => { te = temp1; EmitStringToken(STRNL,  1, 2);    };

    newline;
    space;
*|;

}%%

%% write data nofinal;

#include "parse.mm"

TQNode *TQParseString(NSString *str)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        0, [TQNodeRootBlock new]
    };

    %% write init;
    %% write exec;

    if(p != pe)
        fprintf(stderr, "invalid character '%c'\n", p[0]);

    EmitToken(0); // EOF
    ParseFree(parser, free);
    [parserState.root release];
    return 0;
}

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include "parse.h"
#import <Tranquil/CodeGen/TQNodeBlock.h>

typedef struct {
    int currentLine;
    TQNodeRootBlock *root;
    int lastTokenId;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    Parse(parser, tokenId, val, &parserState); \
    parserState.lastTokenId = tokenId; \
} while(0);

#define EmitToken(tokenId) EmitStringToken(tokenId, 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken((tokenId), [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define LastToken() (parserState.lastTokenId)

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;

    char        = ualpha | '?' | '!';
    constant    = uupper char*;
    identifier  = ulower char*;
    comment     = '\\' [^\n]*;
    newline     = (comment? '\n' space?)+;
    whitespace  = (newline|space|comment);

    lGuillmt    = 0xC2 0xAB; # «
    rGuillmt    = 0xC2 0xBB; # »
    strCont     = '\\' any | (ascii - '"') | ualpha;
    string      = '"' strCont* '"';
    lStr        = '"' strCont* lGuillmt;
    mStr        = rGuillmt strCont* lGuillmt;
    rStr        = rGuillmt strCont* '"';
    selector    = (constant | identifier)? ':';

main := |*
# Hacks
    "{" whitespace* %{ temp1 = p; } identifier %{ temp2 = p; } whitespace* "=" [^>] %{ EmitStringToken(LBRACEDEFARG, 0, 0);   };
    "`" whitespace* %{ temp1 = p; } identifier %{ temp2 = p; } whitespace* "=" [^>] %{ EmitStringToken(BACKTICKDEFARG, 0, 0); };

# Symbols
    "{"                => { EmitToken(LBRACE);                };
    "}" newline        => { EmitToken(RBRACENL);              };
    "}"                => { EmitToken(RBRACE);                };
    "["                => { EmitToken(LBRACKET);              };
    "]" newline        => { EmitToken(RBRACKETNL);            };
    "]"                => { EmitToken(RBRACKET);              };
    "("                => { EmitToken(LPAREN);                };
    ")" newline        => { EmitToken(RPARENNL);              };
    ")"                => { EmitToken(RPAREN);                };
    ","                => { EmitToken(COMMA);                 };
    "`" newline        => { EmitToken(BACKTICKNL); printf("nl\n");};
    "`"                => { EmitToken(BACKTICK);   };
    ";"                => { };
    "="                => { EmitToken(ASSIGN);                };
    "+"                => { EmitToken(PLUS);                  };
    "-"                => { EmitToken(MINUS);                 };
    "--"               => { EmitToken(DECR);                  };
    "--" newline       => { EmitToken(DECRNL);                };
    "++"               => { EmitToken(INCR);                  };
    "++" newline       => { EmitToken(INCRNL);                };
    "*"                => { EmitToken(ASTERISK);              };
    "/"                => { EmitToken(FSLASH);                };
    "~"                => { EmitToken(TILDE);                 };
    "#"                => { EmitToken(HASH);                  };
    "|"                => { EmitToken(PIPE);                  };
    "^"                => { EmitToken(CARET);                 };
    "="                => { EmitToken(ASSIGN);                };
    "=="               => { EmitToken(EQUAL);                 };
    "~="               => { EmitToken(INEQUAL);               };
    "<"                => { EmitToken(LESSER);                };
    ">"                => { EmitToken(GREATER);               };
    "<="               => { EmitToken(LEQUAL);                };
    ">="               => { EmitToken(GEQUAL);                };
    "<<"               => { };
    ">>"               => { };
    "=>"               => { EmitToken(DICTSEP);               };

# Message selectors
    selector           => { EmitStringToken(SELPART, 0, 1);   };

# Keywords
#    "if"               => { EmitStringToken(IF);              };
#    "else"             => { EmitStringToken(ELSE);            };
    "and"|"||"         => { EmitToken(AND);                   };
    "or"|"&&"          => { EmitToken(OR);                    };
#    "while"            => { EmitToken(WHILE);                 };
#    "until"            => { EmitToken(UNTIL);                 };
#    "import"           => { EmitToken(IMPORT);                };
#    "async"            => { EmitToken(ASYNC);                 };
#    "break"            => { EmitToken(BREAK);                 };
#    "skip"             => { EmitToken(SKIP);                  };
    # These are identifier keywords used as expressions => need a *NL variant as well
    "self"    newline  => { EmitStringToken(SELFNL,    0, 0); };
    "super"   newline  => { EmitStringToken(SUPERNL,   0, 0); };
    "..."     newline  => { EmitStringToken(VAARGNL,   0, 0); };
    "nil"     newline  => { EmitStringToken(NILNL,     0, 0); };
    "valid"   newline  => { EmitStringToken(VALIDNL,   0, 0); };
    "yes"     newline  => { EmitStringToken(YESNL,     0, 0); };
    "no"      newline  => { EmitStringToken(NONL,      0, 0); };
    "nothing" newline  => { EmitStringToken(NOTHINGNL, 0, 0); };
    "self"             => { EmitStringToken(SELF,      0, 0); };
    "super"            => { EmitStringToken(SUPER,     0, 0); };
    "..."              => { EmitStringToken(VAARG,     0, 0); };
    "nil"              => { EmitStringToken(NIL,       0, 0); };
    "valid"            => { EmitStringToken(VALID,     0, 0); };
    "yes"              => { EmitStringToken(YES,       0, 0); };
    "no"               => { EmitStringToken(NO,        0, 0); };
    "nothing"          => { EmitStringToken(NOTHING,   0, 0); };

# Identifiers
    identifier newline => { EmitStringToken(IDENTNL, 0, 1);   };
    identifier         => { EmitStringToken(IDENT,   0, 0);   };

    constant newline   => { EmitStringToken(CONSTNL, 0, 1);   };
    constant           => { EmitStringToken(CONST,   0, 0);   };

# Literals
    int   newline      => { EmitIntToken(NUMBERNL, 10, 0);    };
    float newline      => { EmitFloatToken(NUMBERNL);         };
    bin   newline      => { EmitIntToken(NUMBERNL, 2,  2);    };
    oct   newline      => { EmitIntToken(NUMBERNL, 8,  2);    };
    hex   newline      => { EmitIntToken(NUMBERNL, 16, 2);    };

    int                => { EmitIntToken(NUMBER, 10, 0);      };
    float              => { EmitFloatToken(NUMBER);           };
    bin                => { EmitIntToken(NUMBER, 2,  2);      };
    oct                => { EmitIntToken(NUMBER, 8,  2);      };
    hex                => { EmitIntToken(NUMBER, 16, 2);      };

    lStr               => { EmitStringToken(LSTR,   1, 2);    };
    mStr               => { EmitStringToken(MSTR,   2, 2);    };
    rStr               => { EmitStringToken(RSTR,   2, 1);    };
    rStr newline       => { EmitStringToken(RSTRNL, 2, 1);    };
    string             => { EmitStringToken(STR,    1, 2);    };
    string newline     => { EmitStringToken(STRNL,  1, 2);    };

    newline;
    space;
*|;

}%%

%% write data nofinal;

#include "parse.mm"

TQNode *TQParseString(NSString *str)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        0, [TQNodeRootBlock new]
    };

    %% write init;
    %% write exec;

    if(p != pe)
        fprintf(stderr, "invalid character '%c'\n", p[0]);

    EmitToken(0); // EOF
    ParseFree(parser, free);
    [parserState.root release];
    return 0;
}