Fixed a couple of bugs related to set escapes, and improved group and set open/close handling

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . ((backslash . control_sequence) | alpha);

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifier_range_bre  = backslash . range_open . (digit+)? . ','? . (digit+)? .
                          backslash . range_close;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-'? . ([mix]{1,3})?;

  group_ref             = [gk];
  group_name            = alpha . (alnum+)?;
  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");
  group_name_ref        = group_ref . (('<' . group_name . '>') | ("'" . group_name . "'"));

  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_number_ref      = group_ref . (('<' . group_number . '>') | ("'" . group_number . "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error { raise "Premature end of pattern" }

  # group (nesting) and set open/close actions
  action group_opened { group_depth += 1; in_group = true }
  action group_closed { group_depth -= 1; in_group = group_depth > 0 ? true : false }

  action set_opened { set_depth += 1; in_set = true }
  action set_closed { set_depth -= 1; in_set = set_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix @err(premature_end_error) {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    meta_char {
      # this is an ugly workaround. need to review/re-structure the
      # unicode properties machine, which causes this to misbehave.
      if in_set
        self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
      else
        fhold; fgoto main;
      end
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    char_type {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char | [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    property_char > (escaped_set_alpha, 2) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts-1, te)
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        self.emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts-1, te)
      when '\e'; self.emit(:escape, :escape,         text, ts-1, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; self.emit(:escape, :newline,        text, ts-1, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; self.emit(:escape, :space,          text, ts-1, te)
      when '\t'; self.emit(:escape, :tab,            text, ts-1, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts-1, te)
      else
        self.emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_sequence > (backslashed, 3) {
      self.emit(:escape, :meta_sequence, data[ts-1..te-1].pack('c*'), ts-1, te)
    };

    property_char > (escaped_alpha, 2) {
      fhold; fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open %set_opened  {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      # special handling to resolve ambiguity with passive groups
      if data[te]
        c = data[te].chr
        if c == ':' # include the ':'
          self.emit(:group, :options, data[ts..te].pack('c*'), ts, te+1)
          p += 1
        elsif c == ')' # just options by themselves
          self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
        else
          raise "Unexpected '#{c}' in options sequene, expected ':' or ')'"
        end
      else
        raise "Premature end of pattern" unless data[te]
      end
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open @group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Group back-reference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = data[ts..te-1].pack('c*')
      when /\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :name_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          self.emit(:backref, :name_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :number_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          self.emit(:backref, :number_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          self.emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :number_rel_call_sq,  text, ts, te)
        end
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_range  @err(premature_end_error) {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    quantifier_range_bre  @err(premature_end_error) {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group, group_depth = false, 0
    in_set,   set_depth   = false, 0

    %% write init;
    %% write exec;

    raise "Premature end of pattern (missing group closing paranthesis) "+
      "[#{in_group}:#{group_depth}]" if in_group
    raise "Premature end of pattern (missing set closing bracket) "+
      "[#{in_set}:#{set_depth}]" if in_set

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . ((backslash . control_sequence) | alpha);

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifier_range_bre  = backslash . range_open . (digit+)? . ','? . (digit+)? .
                          backslash . range_close;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-'? . ([mix]{1,3})?;

  group_ref             = [gk];
  group_name            = alpha . (alnum+)?;
  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");
  group_name_ref        = group_ref . (('<' . group_name . '>') | ("'" . group_name . "'"));

  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_number_ref      = group_ref . (('<' . group_number . '>') | ("'" . group_number . "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error { raise "Premature end of pattern" }

  # group (nesting) and set open/close actions
  action group_opened { in_group += 1 }
  action group_closed { in_group -= 1 }

  action set_opened { in_set = true }
  action set_closed { in_set = false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix @err(premature_end_error) {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    meta_char {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    char_type {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char | [\\\]\-\,] -- [}] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    property_char > (escaped_set_alpha, 2) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts-1, te)
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        self.emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts-1, te)
      when '\e'; self.emit(:escape, :escape,         text, ts-1, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; self.emit(:escape, :newline,        text, ts-1, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; self.emit(:escape, :space,          text, ts-1, te)
      when '\t'; self.emit(:escape, :tab,            text, ts-1, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts-1, te)
      else
        self.emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_sequence > (backslashed, 3) {
      self.emit(:escape, :meta_sequence, data[ts-1..te-1].pack('c*'), ts-1, te)
    };

    property_char > (escaped_alpha, 2) {
      fhold; fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open %set_opened  {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      # special handling to resolve ambiguity with passive groups
      if data[te]
        c = data[te].chr
        if c == ':' # include the ':'
          self.emit(:group, :options, data[ts..te].pack('c*'), ts, te+1)
          p += 1
        elsif c == ')' # just options by themselves
          self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
        else
          raise "Unexpected '#{c}' in options sequene, expected ':' or ')'"
        end
      else
        raise "Premature end of pattern" unless data[te]
      end
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open @group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Group back-reference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = data[ts..te-1].pack('c*')
      when /\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :name_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          self.emit(:backref, :name_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :number_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          self.emit(:backref, :number_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          self.emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          self.emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          self.emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          self.emit(:backref, :number_rel_call_sq,  text, ts, te)
        end
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_range  @err(premature_end_error) {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    quantifier_range_bre  @err(premature_end_error) {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group = 0
    in_set   = false

    %% write init;
    %% write exec;

    raise "Premature end of pattern (missing group closing paranthesis) [#{in_group}]" if
      in_group > 0
    raise "Premature end of pattern (missing set closing bracket)" if in_set

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner


Fix the "undefined redcloth_to" problem by providing some rdoc for it

/*
 * redcloth_scan.c.rl
 *
 * Copyright (C) 2008 Jason Garber
 */
#define redcloth_scan_c

#include <ruby.h>
#include "redcloth.h"

VALUE mRedCloth, super_ParseError, super_RedCloth, super_HTML, super_LATEX;
int SYM_escape_preformatted, SYM_escape_attributes;

%%{

  machine redcloth_scan;
  include redcloth_common "redcloth_common.c.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  include redcloth_scan "redcloth_scan.rl";

}%%

%% write data nofinal;

VALUE
redcloth_transform(self, p, pe, refs)
  VALUE self;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest;
  char *ts = NULL, *te = NULL, *reg = NULL, *bck = NULL, *eof = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs; CLEAR_REGS()

  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  int list_continue = 0;
  VALUE plain_block; SET_PLAIN_BLOCK("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING_LEN(block) > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return redcloth_transform(self, orig_p, orig_pe, refs_found);
  } else {
    rb_funcall(self, rb_intern("after_transform"), 1, html);
    return html;
  }
}

VALUE
redcloth_transform2(self, str)
  VALUE self, str;
{
  StringValue(str);
  rb_funcall(self, rb_intern("before_transform"), 1, str);
  return redcloth_transform(self, RSTRING_PTR(str), RSTRING_PTR(str) + RSTRING_LEN(str) + 1, Qnil);
}

/*
 * Converts special characters into HTML entities.
 */
static VALUE
redcloth_html_esc(int argc, VALUE* argv, VALUE self) //(self, str, level)
{
  VALUE str, level;

  rb_scan_args(argc, argv, "11", &str, &level);

  VALUE new_str = rb_str_new2("");
  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return;

  while (t2 < te) {
    ch = NULL;

    // normal + pre
    switch (*t2)
    {
      case '&':  ch = "amp";    break;
      case '>':  ch = "gt";     break;
      case '<':  ch = "lt";     break;
    }

    // normal (non-pre)
    if (level != SYM_escape_preformatted) {
      switch (*t2)
      {
        case '\n': ch = "br";     break;
        case '"' : ch = "quot";   break;
        case '\'':
          ch = (level == SYM_escape_attributes) ? "apos" : "squot";
          break;
      }
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      rb_str_concat(new_str, rb_funcall(self, rb_intern(ch), 1, rb_hash_new()));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Converts special characters into LaTeX entities.
 */
static VALUE
redcloth_latex_esc(VALUE self, VALUE str)
{
  VALUE new_str = rb_str_new2("");

  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return;

  while (t2 < te) {
    ch = NULL;

    switch (*t2)
    {
      case '{':  ch = "#123";   break;
      case '}':  ch = "#125";   break;
      case '\\': ch = "#92";    break;
      case '#':  ch = "#35";    break;
      case '$':  ch = "#36";    break;
      case '%':  ch = "#37";    break;
      case '&':  ch = "amp";    break;
      case '_':  ch = "#95";    break;
      case '^':  ch = "circ";   break;
      case '~':  ch = "tilde";  break;
      case '<':  ch = "lt";     break;
      case '>':  ch = "gt";     break;
      case '\n': ch = "#10";    break;
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      VALUE opts = rb_hash_new();
      rb_hash_aset(opts, ID2SYM(rb_intern("text")), rb_str_new2(ch));
      rb_str_concat(new_str, rb_funcall(self, rb_intern("entity"), 1, opts));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Transforms a Textile document with +formatter+
 */
static VALUE
redcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  rb_funcall(self, rb_intern("delete!"), 1, rb_str_new2("\r"));
  VALUE working_copy = rb_obj_clone(self);
  rb_extend_object(working_copy, formatter);
  if (rb_funcall(working_copy, rb_intern("lite_mode"), 0) == Qtrue) {
    return redcloth_inline2(working_copy, self, rb_hash_new());
  } else {
    return redcloth_transform2(working_copy, self);
  }
}

void Init_redcloth_scan()
{
  mRedCloth = rb_define_module("RedCloth");
  /* A Textile document that can be converted to other formats. See
   the README for Textile syntax. */
  super_RedCloth = rb_define_class_under(mRedCloth, "TextileDoc", rb_cString);
  rb_define_method(super_RedCloth, "to", redcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  /* Escaping */
  rb_define_method(super_RedCloth, "html_esc", redcloth_html_esc, -1);
  rb_define_method(super_RedCloth, "latex_esc", redcloth_latex_esc, 1);
  SYM_escape_preformatted   = ID2SYM(rb_intern("html_escape_preformatted"));
  SYM_escape_attributes     = ID2SYM(rb_intern("html_escape_attributes"));
}

/*
 * redcloth_scan.c.rl
 *
 * Copyright (C) 2008 Jason Garber
 */
#define redcloth_scan_c

#include <ruby.h>
#include "redcloth.h"

VALUE mRedCloth, super_ParseError, super_RedCloth, super_HTML, super_LATEX;
int SYM_escape_preformatted, SYM_escape_attributes;

%%{

  machine redcloth_scan;
  include redcloth_common "redcloth_common.c.rl";

  action extend { extend = rb_hash_aref(regs, ID2SYM(rb_intern("type"))); }

  include redcloth_scan "redcloth_scan.rl";

}%%

%% write data nofinal;

VALUE
redcloth_transform(self, p, pe, refs)
  VALUE self;
  char *p, *pe;
  VALUE refs;
{
  char *orig_p = p, *orig_pe = pe;
  int cs, act, nest;
  char *ts = NULL, *te = NULL, *reg = NULL, *bck = NULL, *eof = NULL;
  VALUE html = rb_str_new2("");
  VALUE table = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs; CLEAR_REGS()

  VALUE list_layout = Qnil;
  char *list_type = NULL;
  VALUE list_index = rb_ary_new();
  int list_continue = 0;
  VALUE plain_block; SET_PLAIN_BLOCK("p");
  VALUE extend = Qnil;
  char listm[10] = "";
  VALUE refs_found = rb_hash_new();

  %% write init;

  %% write exec;

  if (RSTRING_LEN(block) > 0)
  {
    ADD_BLOCK();
  }

  if ( NIL_P(refs) && rb_funcall(refs_found, rb_intern("empty?"), 0) == Qfalse ) {
    return redcloth_transform(self, orig_p, orig_pe, refs_found);
  } else {
    rb_funcall(self, rb_intern("after_transform"), 1, html);
    return html;
  }
}

VALUE
redcloth_transform2(self, str)
  VALUE self, str;
{
  StringValue(str);
  rb_funcall(self, rb_intern("before_transform"), 1, str);
  return redcloth_transform(self, RSTRING_PTR(str), RSTRING_PTR(str) + RSTRING_LEN(str) + 1, Qnil);
}

/*
 * Converts special characters into HTML entities.
 */
static VALUE
redcloth_html_esc(int argc, VALUE* argv, VALUE self) //(self, str, level)
{
  VALUE str, level;

  rb_scan_args(argc, argv, "11", &str, &level);

  VALUE new_str = rb_str_new2("");
  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return;

  while (t2 < te) {
    ch = NULL;

    // normal + pre
    switch (*t2)
    {
      case '&':  ch = "amp";    break;
      case '>':  ch = "gt";     break;
      case '<':  ch = "lt";     break;
    }

    // normal (non-pre)
    if (level != SYM_escape_preformatted) {
      switch (*t2)
      {
        case '\n': ch = "br";     break;
        case '"' : ch = "quot";   break;
        case '\'':
          ch = (level == SYM_escape_attributes) ? "apos" : "squot";
          break;
      }
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      rb_str_concat(new_str, rb_funcall(self, rb_intern(ch), 1, rb_hash_new()));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

/*
 * Converts special characters into LaTeX entities.
 */
static VALUE
redcloth_latex_esc(VALUE self, VALUE str)
{
  VALUE new_str = rb_str_new2("");

  if (str == Qnil)
    return new_str;

  StringValue(str);

  if (RSTRING_LEN(str) == 0)
    return new_str;

  char *ts = RSTRING_PTR(str), *te = RSTRING_PTR(str) + RSTRING_LEN(str);
  char *t = ts, *t2 = ts, *ch = NULL;
  if (te <= ts) return;

  while (t2 < te) {
    ch = NULL;

    switch (*t2)
    {
      case '{':  ch = "#123";   break;
      case '}':  ch = "#125";   break;
      case '\\': ch = "#92";    break;
      case '#':  ch = "#35";    break;
      case '$':  ch = "#36";    break;
      case '%':  ch = "#37";    break;
      case '&':  ch = "amp";    break;
      case '_':  ch = "#95";    break;
      case '^':  ch = "circ";   break;
      case '~':  ch = "tilde";  break;
      case '<':  ch = "lt";     break;
      case '>':  ch = "gt";     break;
      case '\n': ch = "#10";    break;
    }

    if (ch != NULL)
    {
      if (t2 > t)
        rb_str_cat(new_str, t, t2-t);
      VALUE opts = rb_hash_new();
      rb_hash_aset(opts, ID2SYM(rb_intern("text")), rb_str_new2(ch));
      rb_str_concat(new_str, rb_funcall(self, rb_intern("entity"), 1, opts));
      t = t2 + 1;
    }

    t2++;
  }
  if (t2 > t)
    rb_str_cat(new_str, t, t2-t);

  return new_str;
}

static VALUE
redcloth_to(self, formatter)
  VALUE self, formatter;
{
  char *pe, *p;
  int len = 0;

  rb_funcall(self, rb_intern("delete!"), 1, rb_str_new2("\r"));
  VALUE working_copy = rb_obj_clone(self);
  rb_extend_object(working_copy, formatter);
  if (rb_funcall(working_copy, rb_intern("lite_mode"), 0) == Qtrue) {
    return redcloth_inline2(working_copy, self, rb_hash_new());
  } else {
    return redcloth_transform2(working_copy, self);
  }
}

void Init_redcloth_scan()
{
  mRedCloth = rb_define_module("RedCloth");
  /* A Textile document that can be converted to other formats. See
   the README for Textile syntax. */
  super_RedCloth = rb_define_class_under(mRedCloth, "TextileDoc", rb_cString);
  rb_define_method(super_RedCloth, "to", redcloth_to, 1);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
  /* Escaping */
  rb_define_method(super_RedCloth, "html_esc", redcloth_html_esc, -1);
  rb_define_method(super_RedCloth, "latex_esc", redcloth_latex_esc, 1);
  SYM_escape_preformatted   = ID2SYM(rb_intern("html_escape_preformatted"));
  SYM_escape_attributes     = ID2SYM(rb_intern("html_escape_attributes"));
}


Remove unused p-variable in C parser state

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define MARK_STR_NEW() rb_external_str_new(mark, MARK_LEN)

#define true 1
#define false 0

typedef struct {
  VALUE error;
  long max_message_size;

  VALUE chunk;
  const char *p;
  int cs;
  const char *mark;
  VALUE mark_key;
  VALUE mark_message;
  long mark_message_size;
  long mark_content_length;
} parser_state_t;

VALUE mStomp = Qnil;
VALUE cMessage = Qnil;
VALUE eMessageSizeExceeded = Qnil;
ID g_new;
ID g_write_command;
ID g_write_header;
ID g_write_body;
ID g_content_length;
ID g_build_parse_error;
ID g_max_message_size;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = rb_funcall(cMessage, g_new, 2, Qnil, Qnil);
    mark_message_size = 0;
  }

  action write_command {
    rb_funcall(mark_message, g_write_command, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action mark_key {
    mark_key = MARK_STR_NEW();
    mark = NULL;
  }

  action write_header {
    rb_funcall(mark_message, g_write_header, 2, mark_key, MARK_STR_NEW());
    mark_key = Qnil;
    mark = NULL;
  }

  action finish_headers {
    VALUE length = rb_funcall(mark_message, g_content_length, 0);
    if ( ! NIL_P(length)) {
      mark_content_length = NUM2LONG(length);
    } else {
      mark_content_length = -1;
    }
  }

  action write_body {
    rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action consume_null {
    ((mark_content_length != -1) && (MARK_LEN < mark_content_length))
  }

  action consume_octet {
    ((mark_content_length == -1) || (MARK_LEN < mark_content_length))
  }

  action check_message_size {
    mark_message_size += 1;
    if (mark_message_size > max_message_size) {
      rb_raise(eMessageSizeExceeded, "");
    }
  }

  action finish_message {
    rb_yield(mark_message);
    mark_message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->error);
  rb_gc_mark(state->mark_key);
  rb_gc_mark(state->mark_message);
  rb_gc_mark(state->chunk);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  VALUE max_message_size;
  rb_scan_args(argc, argv, "01", &max_message_size);

  if (max_message_size == Qnil) {
    max_message_size = rb_funcall(mStomp, g_max_message_size, 0);
  }

  state->error = Qnil;
  state->max_message_size = FIX2LONG(max_message_size);
  state->chunk = Qnil;
  state->cs = start;
  state->mark = NULL;
  state->mark_key = Qnil;
  state->mark_message = Qnil;
  state->mark_message_size = 0;
  state->mark_content_length = 0;

  return self;
}

static VALUE parser_parse(VALUE self, VALUE new_chunk) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  if (NIL_P(state->error)) {
    VALUE chunk = Qnil;
    const char *p = NULL;
    const char *mark = NULL;

    if ( ! NIL_P(state->chunk)) {
      long offset = RSTRING_LEN(state->chunk);
      long mark_offset = state->mark - RSTRING_PTR(state->chunk);

      chunk = rb_str_append(state->chunk, new_chunk);
      p = RSTRING_PTR(chunk) + offset;
      mark = RSTRING_PTR(chunk) + mark_offset;
    } else {
      chunk = new_chunk;
      p = RSTRING_PTR(chunk);
    }

    const char *pe = RSTRING_END(chunk);
    long max_message_size = state->max_message_size;

    int cs = state->cs;
    VALUE mark_key = state->mark_key;
    VALUE mark_message = state->mark_message;
    long mark_message_size = state->mark_message_size;
    long mark_content_length = state->mark_content_length;

    %% write exec;

    if (mark != NULL) {
      state->chunk = chunk;
    } else {
      state->chunk = Qnil;
    }

    state->cs = cs;
    state->mark = mark;
    state->mark_key = mark_key;
    state->mark_message = mark_message;
    state->mark_message_size = mark_message_size;
    state->mark_content_length = mark_content_length;

    if (cs == error) {
      long index = p - RSTRING_PTR(chunk);
      state->error = rb_funcall(mStomp, g_build_parse_error, 2, chunk, LONG2NUM(index));
    }
  }

  if ( ! NIL_P(state->error)) {
    rb_exc_raise(state->error);
  }

  return Qnil;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));

  mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));
  eMessageSizeExceeded = rb_const_get(mStomp, rb_intern("MessageSizeExceeded"));

  g_new = rb_intern("new");
  g_write_command = rb_intern("write_command");
  g_write_header = rb_intern("write_header");
  g_write_body = rb_intern("write_body");
  g_content_length = rb_intern("content_length");
  g_build_parse_error = rb_intern("build_parse_error");
  g_max_message_size = rb_intern("max_message_size");

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "parse", parser_parse, 1);
}

#include <ruby.h>

#if DEBUG_H
#  define DEBUG(fmt, ...) do { fprintf(stderr, fmt "\n", ##__VA_ARGS__); } while(0)
#else
#  define DEBUG(...)
#endif

#define UNUSED(x) (void)(x)
#define MARK_LEN (p - mark)
#define MARK_STR_NEW() rb_external_str_new(mark, MARK_LEN)

#define true 1
#define false 0

typedef struct {
  VALUE error;
  long max_message_size;

  VALUE chunk;
  const char *p;
  int cs;
  const char *mark;
  VALUE mark_key;
  VALUE mark_message;
  long mark_message_size;
  long mark_content_length;
} parser_state_t;

VALUE mStomp = Qnil;
VALUE cMessage = Qnil;
VALUE eMessageSizeExceeded = Qnil;
ID g_new;
ID g_write_command;
ID g_write_header;
ID g_write_body;
ID g_content_length;
ID g_build_parse_error;
ID g_max_message_size;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = rb_funcall(cMessage, g_new, 2, Qnil, Qnil);
    mark_message_size = 0;
  }

  action write_command {
    rb_funcall(mark_message, g_write_command, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action mark_key {
    mark_key = MARK_STR_NEW();
    mark = NULL;
  }

  action write_header {
    rb_funcall(mark_message, g_write_header, 2, mark_key, MARK_STR_NEW());
    mark_key = Qnil;
    mark = NULL;
  }

  action finish_headers {
    VALUE length = rb_funcall(mark_message, g_content_length, 0);
    if ( ! NIL_P(length)) {
      mark_content_length = NUM2LONG(length);
    } else {
      mark_content_length = -1;
    }
  }

  action write_body {
    rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());
    mark = NULL;
  }

  action consume_null {
    ((mark_content_length != -1) && (MARK_LEN < mark_content_length))
  }

  action consume_octet {
    ((mark_content_length == -1) || (MARK_LEN < mark_content_length))
  }

  action check_message_size {
    mark_message_size += 1;
    if (mark_message_size > max_message_size) {
      rb_raise(eMessageSizeExceeded, "");
    }
  }

  action finish_message {
    rb_yield(mark_message);
    mark_message = Qnil;
  }

  include message_common "parser_common.rl";

  write data noprefix;
}%%

static void parser_free(parser_state_t *state) {
  // TODO: free memory inside struct!
  xfree(state);
}

static void parser_mark(parser_state_t *state) {
  rb_gc_mark(state->error);
  rb_gc_mark(state->mark_key);
  rb_gc_mark(state->mark_message);
  rb_gc_mark(state->chunk);
}

static VALUE parser_alloc(VALUE klass) {
  parser_state_t *state = ALLOC(parser_state_t);
  return Data_Wrap_Struct(klass, parser_mark, parser_free, state);
}

static VALUE parser_initialize(int argc, VALUE *argv, VALUE self) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  VALUE max_message_size;
  rb_scan_args(argc, argv, "01", &max_message_size);

  if (max_message_size == Qnil) {
    max_message_size = rb_funcall(mStomp, g_max_message_size, 0);
  }

  state->error = Qnil;
  state->max_message_size = FIX2LONG(max_message_size);
  state->chunk = Qnil;
  state->p = NULL;
  state->cs = start;
  state->mark = NULL;
  state->mark_key = Qnil;
  state->mark_message = Qnil;
  state->mark_message_size = 0;
  state->mark_content_length = 0;

  return self;
}

static VALUE parser_parse(VALUE self, VALUE new_chunk) {
  parser_state_t *state;
  Data_Get_Struct(self, parser_state_t, state);

  if (NIL_P(state->error)) {
    VALUE chunk = Qnil;
    const char *p = NULL;
    const char *mark = NULL;

    if ( ! NIL_P(state->chunk)) {
      long offset = RSTRING_LEN(state->chunk);
      long mark_offset = state->mark - RSTRING_PTR(state->chunk);

      chunk = rb_str_append(state->chunk, new_chunk);
      p = RSTRING_PTR(chunk) + offset;
      mark = RSTRING_PTR(chunk) + mark_offset;
    } else {
      chunk = new_chunk;
      p = RSTRING_PTR(chunk);
    }

    const char *pe = RSTRING_END(chunk);
    long max_message_size = state->max_message_size;

    int cs = state->cs;
    VALUE mark_key = state->mark_key;
    VALUE mark_message = state->mark_message;
    long mark_message_size = state->mark_message_size;
    long mark_content_length = state->mark_content_length;

    %% write exec;

    if (mark != NULL) {
      state->chunk = chunk;
    } else {
      state->chunk = Qnil;
    }

    state->cs = cs;
    state->mark = mark;
    state->mark_key = mark_key;
    state->mark_message = mark_message;
    state->mark_message_size = mark_message_size;
    state->mark_content_length = mark_content_length;

    if (cs == error) {
      long index = p - RSTRING_PTR(chunk);
      state->error = rb_funcall(mStomp, g_build_parse_error, 2, chunk, LONG2NUM(index));
    }
  }

  if ( ! NIL_P(state->error)) {
    rb_exc_raise(state->error);
  }

  return Qnil;
}

void Init_c_parser(void) {
  VALUE mStompede = rb_const_get(rb_cObject, rb_intern("Stompede"));

  mStomp = rb_const_get(mStompede, rb_intern("Stomp"));
  cMessage = rb_const_get(mStomp, rb_intern("Message"));
  eMessageSizeExceeded = rb_const_get(mStomp, rb_intern("MessageSizeExceeded"));

  g_new = rb_intern("new");
  g_write_command = rb_intern("write_command");
  g_write_header = rb_intern("write_header");
  g_write_body = rb_intern("write_body");
  g_content_length = rb_intern("content_length");
  g_build_parse_error = rb_intern("build_parse_error");
  g_max_message_size = rb_intern("max_message_size");

  VALUE cParser = rb_define_class_under(mStomp, "CParser", rb_cObject);
  rb_define_alloc_func(cParser, parser_alloc);

  rb_define_method(cParser, "initialize", parser_initialize, -1);
  rb_define_method(cParser, "parse", parser_parse, 1);
}


The range keyword has been disabled for the parser rewrite of 5.17 so that the ragel language matches 5.16 exactly. Can revert these changes to the test to make diffing against the 5.16 output easier. Will enable again in the future.

/*
 * @LANG: c
 */

#include <stdio.h>
#include <string.h>

struct range
{
	int cs;
};

%%{
	machine range;
	variable curstate fsm->cs;

	main := ( 'a' .. 'c' | 'c' .. 'e' | 'm' .. 'n' | 'a' .. 'z' ) '\n';
}%%

%% write data;

void range_init( struct range *fsm )
{
	%% write init;
}

void range_execute( struct range *fsm, const char *_data, int _len )
{
	const char *p = _data;
	const char *pe = _data+_len;

	%% write exec;
}

int range_finish( struct range *fsm )
{
	%% write eof;

	if ( fsm->cs == range_error )
		return -1;
	if ( fsm->cs >= range_first_final )
		return 1;
	return 0;
}

struct range fsm;

void test( char *buf )
{
	int len = strlen( buf );
	range_init( &fsm );
	range_execute( &fsm, buf, len );
	if ( range_finish( &fsm ) > 0 )
		printf("ACCEPT\n");
	else
		printf("FAIL\n");
}

int main()
{
	test( "a\n" );
	test( "z\n" );
	test( "g\n" );
	test( "no\n" );
	test( "1\n" );

	return 0;
}

#ifdef _____OUTPUT_____
ACCEPT
ACCEPT
ACCEPT
FAIL
FAIL
#endif

/*
 * @LANG: c
 */

#include <stdio.h>
#include <string.h>

struct range
{
	int cs;
};

%%{
	machine range_fsm;
	variable curstate fsm->cs;

	main := ( 'a' .. 'c' | 'c' .. 'e' | 'm' .. 'n' | 'a' .. 'z' ) '\n';
}%%

%% write data;

void range_init( struct range *fsm )
{
	%% write init;
}

void range_execute( struct range *fsm, const char *_data, int _len )
{
	const char *p = _data;
	const char *pe = _data+_len;

	%% write exec;
}

int range_finish( struct range *fsm )
{
	%% write eof;

	if ( fsm->cs == range_fsm_error )
		return -1;
	if ( fsm->cs >= range_fsm_first_final )
		return 1;
	return 0;
}

struct range fsm;

void test( char *buf )
{
	int len = strlen( buf );
	range_init( &fsm );
	range_execute( &fsm, buf, len );
	if ( range_finish( &fsm ) > 0 )
		printf("ACCEPT\n");
	else
		printf("FAIL\n");
}

int main()
{
	test( "a\n" );
	test( "z\n" );
	test( "g\n" );
	test( "no\n" );
	test( "1\n" );

	return 0;
}

#ifdef _____OUTPUT_____
ACCEPT
ACCEPT
ACCEPT
FAIL
FAIL
#endif


[jnc_test] removed old-style %% regexs from the syntax highlighter

//..............................................................................
//
//  This file is part of the Jancy toolkit.
//
//  Jancy is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/jancy/license.txt
//
//..............................................................................

%%{

machine jancy_lexer;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# standard definitions
#

dec    = [0-9];
hex    = [0-9a-fA-F];
oct    = [0-7];
bin    = [01];
id     = [_a-zA-Z] [_a-zA-Z0-9]*;
ws     = [ \t\r]+;
nl     = '\n';
lc_nl  = '\\' '\r'? nl;
esc    = '\\' [^\n];
lit_dq = '"' ([^"\n\\] | esc)* (["\\] | nl);
lit_sq = "'" ([^'\n\\] | esc)* (['\\] | nl);

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# main machine
#

main := |*

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

(
'import'          |
'namespace'       |
'extension'       |
'dynamiclib'      |
'using'           |
'friend'          |
'public'          |
'protected'       |
'alignment'       |
'setas'           |

'typedef'         |
'alias'           |
'static'          |
'threadlocal'     |
'stack'           |
'heap'            |
'abstract'        |
'virtual'         |
'override'        |
'mutable'         |
'disposable'      |

'signed'          |
'unsigned'        |
'bigendian'       |
'const'           |
'readonly'        |
'volatile'        |
'safe'            |
'unsafe'          |
'weak'            |
'thin'            |
'errorcode'       |
'cdecl'           |
'stdcall'         |
'thiscall'        |
'jnccall'         |
'array'           |
'function'        |
'property'        |
'bindable'        |
'autoget'         |
'indexed'         |
'multicast'       |
'event'           |
'reactor'         |

'auto'            |
'anydata'         |
'void'            |
'variant'         |
'bool'            |
'int'             |
'intptr'          |
'char'            |
'short'           |
'long'            |
'float'           |
'double'          |
'enum'            |
'struct'          |
'union'           |
'class'           |
'opaque'          |
'exposed'         |
'bitflag'         |

'get'             |
'set'             |
'preconstruct'    |
'construct'       |
'destruct'        |
'operator'        |
'postfix'         |

'if'              |
'else'            |
'for'             |
'while'           |
'do'              |
'break'           |
'break' [1-9]?    |
'continue' [1-9]? |
'return'          |
'switch'          |
'case'            |
'default'         |
'once'            |
'onevent'         |
'try'             |
'throw'           |
'catch'           |
'finally'         |
'nestedscope'     |
'assert'          |
'regex'           |

'basetype' [1-9]? |
'this'            |
'true'            |
'false'           |
'null'            |

'new'             |
'delete'          |
'sizeof'          |
'countof'         |
'offsetof'        |
'typeof'          |
'bindingof'       |
'dynamic'

)                   { colorize (ts, te, Qt::blue); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

id                  ;
(lit_sq | lit_dq)   { colorize (ts, te, Qt::darkRed); };
dec+                { colorize (ts, te, Qt::darkRed); };
'0' oct+            { colorize (ts, te, Qt::darkRed); };
'0' [xX] hex+       { colorize (ts, te, Qt::darkRed); };
'0' [oO] oct+       { colorize (ts, te, Qt::darkRed); };
'0' [bB] bin+       { colorize (ts, te, Qt::darkRed); };
'0' [nNdD] dec+     { colorize (ts, te, Qt::darkRed); };
'0' [xXoObBnNdD] lit_dq
					{ colorize (ts, te, Qt::darkRed); };
dec+ ('.' dec*) | ([eE] [+\-]? dec+)
					{ colorize (ts, te, Qt::darkRed); };
'$' lit_dq          { colorize (ts, te, Qt::darkRed); };

'"""'               { colorize (ts, te, Qt::darkRed); fgoto lit_ml; };
'0' [xXoObBnNdD] '"""'
					{ colorize (ts, te, Qt::darkRed); fgoto lit_ml; };

'//' any*           { colorize (ts, te, Qt::darkGray); };
'/*'                { colorize (ts, te, Qt::darkGray); fgoto comment; };

ws | nl             ;
any                 ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# comment machine
#

comment := |*

'*/'                { colorize (ts, te, Qt::darkGray); fgoto main; };
any                 { colorize (ts, te, Qt::darkGray); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# multi-line literal machine
#

lit_ml := |*

'"""'               { colorize (ts, te, Qt::darkRed); fgoto main; };
any                 { colorize (ts, te, Qt::darkRed); };

*|;

}%%

//..............................................................................

#define BLOCK_STATE_NONE	0
#define BLOCK_STATE_COMMENT 1
#define BLOCK_STATE_LIT_ML  2

void JancyHighlighter::ragelInit ()
{
	%% write init;
}

void JancyHighlighter::ragelExec ()
{
	%% write exec;
}

void JancyHighlighter::ragelExecPreEvent (int &ragelState)
{
	setCurrentBlockState (BLOCK_STATE_NONE);

	int prevBlockState = previousBlockState ();
	switch (prevBlockState)
	{
	case BLOCK_STATE_COMMENT:
		ragelState = jancy_lexer_en_comment;
		break;

	case BLOCK_STATE_LIT_ML:
		ragelState = jancy_lexer_en_lit_ml;
		break;
	}
}

void JancyHighlighter::ragelExecPostEvent (int ragelState)
{
	switch (ragelState)
	{
	case jancy_lexer_en_comment:
		setCurrentBlockState (BLOCK_STATE_COMMENT);
		break;

	case jancy_lexer_en_lit_ml:
		setCurrentBlockState (BLOCK_STATE_LIT_ML);
		break;
	}
}

//..............................................................................

//..............................................................................
//
//  This file is part of the Jancy toolkit.
//
//  Jancy is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/jancy/license.txt
//
//..............................................................................

%%{

machine jancy_lexer;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# standard definitions
#

dec    = [0-9];
hex    = [0-9a-fA-F];
oct    = [0-7];
bin    = [01];
id     = [_a-zA-Z] [_a-zA-Z0-9]*;
ws     = [ \t\r]+;
nl     = '\n';
lc_nl  = '\\' '\r'? nl;
esc    = '\\' [^\n];
lit_dq = '"' ([^"\n\\] | esc)* (["\\] | nl);
lit_sq = "'" ([^'\n\\] | esc)* (['\\] | nl);

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# main machine
#

main := |*

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

(
'import'          |
'namespace'       |
'extension'       |
'dynamiclib'      |
'using'           |
'friend'          |
'public'          |
'protected'       |
'alignment'       |
'setas'           |

'typedef'         |
'alias'           |
'static'          |
'threadlocal'     |
'stack'           |
'heap'            |
'abstract'        |
'virtual'         |
'override'        |
'mutable'         |
'disposable'      |

'signed'          |
'unsigned'        |
'bigendian'       |
'const'           |
'readonly'        |
'volatile'        |
'safe'            |
'unsafe'          |
'weak'            |
'thin'            |
'errorcode'       |
'cdecl'           |
'stdcall'         |
'thiscall'        |
'jnccall'         |
'array'           |
'function'        |
'property'        |
'bindable'        |
'autoget'         |
'indexed'         |
'multicast'       |
'event'           |
'reactor'         |

'auto'            |
'anydata'         |
'void'            |
'variant'         |
'bool'            |
'int'             |
'intptr'          |
'char'            |
'short'           |
'long'            |
'float'           |
'double'          |
'enum'            |
'struct'          |
'union'           |
'class'           |
'opaque'          |
'exposed'         |
'bitflag'         |

'get'             |
'set'             |
'preconstruct'    |
'construct'       |
'destruct'        |
'operator'        |
'postfix'         |

'if'              |
'else'            |
'for'             |
'while'           |
'do'              |
'break'           |
'break' [1-9]?    |
'continue' [1-9]? |
'return'          |
'switch'          |
'case'            |
'default'         |
'once'            |
'onevent'         |
'try'             |
'throw'           |
'catch'           |
'finally'         |
'nestedscope'     |
'assert'          |
'regex'           |

'basetype' [1-9]? |
'this'            |
'true'            |
'false'           |
'null'            |

'new'             |
'delete'          |
'sizeof'          |
'countof'         |
'offsetof'        |
'typeof'          |
'bindingof'       |
'dynamic'

)                   { colorize (ts, te, Qt::blue); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

id                  ;
(lit_sq | lit_dq)   { colorize (ts, te, Qt::darkRed); };
dec+                { colorize (ts, te, Qt::darkRed); };
'0' oct+            { colorize (ts, te, Qt::darkRed); };
'0' [xX] hex+       { colorize (ts, te, Qt::darkRed); };
'0' [oO] oct+       { colorize (ts, te, Qt::darkRed); };
'0' [bB] bin+       { colorize (ts, te, Qt::darkRed); };
'0' [nNdD] dec+     { colorize (ts, te, Qt::darkRed); };
'0' [xXoObBnNdD] lit_dq
					{ colorize (ts, te, Qt::darkRed); };
dec+ ('.' dec*) | ([eE] [+\-]? dec+)
					{ colorize (ts, te, Qt::darkRed); };
'$' lit_dq          { colorize (ts, te, Qt::darkRed); };

'%%'                { colorize (ts, te, Qt::darkRed); fgoto regexp; };
'"""'               { colorize (ts, te, Qt::darkRed); fgoto lit_ml; };
'0' [xXoObBnNdD] '"""'
					{ colorize (ts, te, Qt::darkRed); fgoto lit_ml; };

'//' any*           { colorize (ts, te, Qt::darkGray); };
'/*'                { colorize (ts, te, Qt::darkGray); fgoto comment; };

ws | nl             ;
any                 ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# comment machine
#

comment := |*

'*/'                { colorize (ts, te, Qt::darkGray); fgoto main; };
any                 { colorize (ts, te, Qt::darkGray); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# regexp machine
#

regexp := |*

any* '\\'           { colorize (ts, te, Qt::darkRed); };
any*                { colorize (ts, te, Qt::darkRed); fgoto main; };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# multi-line literal machine
#

lit_ml := |*

'"""'               { colorize (ts, te, Qt::darkRed); fgoto main; };
any                 { colorize (ts, te, Qt::darkRed); };

*|;

}%%

//..............................................................................

#define BLOCK_STATE_NONE	0
#define BLOCK_STATE_COMMENT 1
#define BLOCK_STATE_REGEXP  2
#define BLOCK_STATE_LIT_ML  3

void JancyHighlighter::ragelInit ()
{
	%% write init;
}

void JancyHighlighter::ragelExec ()
{
	%% write exec;
}

void JancyHighlighter::ragelExecPreEvent (int &ragelState)
{
	setCurrentBlockState (BLOCK_STATE_NONE);

	int prevBlockState = previousBlockState ();
	switch (prevBlockState)
	{
	case BLOCK_STATE_COMMENT:
		ragelState = jancy_lexer_en_comment;
		break;

	case BLOCK_STATE_REGEXP:
		ragelState = jancy_lexer_en_regexp;
		break;

	case BLOCK_STATE_LIT_ML:
		ragelState = jancy_lexer_en_lit_ml;
		break;
	}
}

void JancyHighlighter::ragelExecPostEvent (int ragelState)
{
	switch (ragelState)
	{
	case jancy_lexer_en_comment:
		setCurrentBlockState (BLOCK_STATE_COMMENT);
		break;

	case jancy_lexer_en_regexp:
		setCurrentBlockState (BLOCK_STATE_REGEXP);
		break;

	case jancy_lexer_en_lit_ml:
		setCurrentBlockState (BLOCK_STATE_LIT_ML);
		break;
	}
}

//..............................................................................