Remove some duplicate whitespace

Remove some duplicate whitespace

%%{
  machine re_scanner;
  include re_char_type "char_type.rl";
  include re_property  "property.rl";

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
  utf8_multibyte        = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parentheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  comment               = ('#' . [^\n]* . '\n'?);

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby at the moment
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrtv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;

  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;

  sequence_char         = [CMcux];

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantity_exact        = (digit+);
  quantity_minimum      = (digit+) . ',';
  quantity_maximum      = ',' . (digit+);
  quantity_range        = (digit+) . ',' . (digit+);
  quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
                          quantity_maximum | quantity_range ) . range_close .
                          quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  conditional           = '(?(';

  group_comment         = '?#' . [^)]* . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_absence         = '?~';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  # try to treat every other group head as options group, like Ruby
  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;

  group_ref             = [gk];
  group_name_id_ab      = ([^0-9\->] | utf8_multibyte) . ([^>] | utf8_multibyte)*;
  group_name_id_sq      = ([^0-9\-'] | utf8_multibyte) . ([^'] | utf8_multibyte)*;
  group_number          = '-'? . [1-9] . [0-9]*;
  group_level           = [+\-] . [0-9]+;

  group_name            = ('<' . group_name_id_ab? . '>') |
                          ("'" . group_name_id_sq? . "'");
  group_lookup          = group_name | group_number;

  group_named           = ('?' . group_name );

  group_name_backref    = 'k' . (('<' . group_name_id_ab? . group_level? '>') |
                                 ("'" . group_name_id_sq? . group_level? "'"));
  group_name_call       = 'g' . (('<' . group_name_id_ab? . group_level? '>') |
                                 ("'" . group_name_id_sq? . group_level? "'"));

  group_number_backref  = 'k' . (('<' . group_number . group_level? '>') |
                                 ("'" . group_number . group_level? "'"));
  group_number_call     = 'g' . (('<' . ((group_number . group_level?) | '0') '>') |
                                 ("'" . ((group_number . group_level?) | '0') "'"));

  group_type            = group_atomic | group_passive | group_absence | group_named;

  keep_mark             = 'K';

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parentheses | brackets |
                          line_anchor | quantifier_greedy;

  literal_delimiters    = ']' | '}';

  ascii_print           = ((0x20..0x7e) - meta_char - '#');
  ascii_nonprint        = (0x01..0x1f | 0x7f);

  non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
                          keep_mark | sequence_char;

  # escapes that also work within a character set
  set_escape            = backslash | brackets | escaped_ascii | property_char |
                          sequence_char | single_codepoint_char_type;

  # EOF error, used where it can be detected
  action premature_end_error {
    text = copy(data, ts ? ts-1 : 0, -1)
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = copy(data, ts ? ts-1 : 0, -1)
    validation_error(:sequence, 'sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { self.group_depth = group_depth + 1 }
  action group_closed { self.group_depth = group_depth - 1 }
  action set_opened   { self.set_depth   = set_depth   + 1 }
  action set_closed   { self.set_depth   = set_depth   - 1 }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    set_close > (set_meta, 2) @set_closed {
      emit(:set, :close, copy(data, ts, te))
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-]' @set_closed { # special case, emits two tokens
      emit(:literal, :literal, copy(data, ts, te-1))
      emit(:set, :close, copy(data, ts+1, te))
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-&&' { # special case, emits two tokens
      emit(:literal, :literal, '-')
      emit(:set, :intersection, '&&')
    };

    '^' {
      text = copy(data, ts, te)
      if tokens.last[1] == :open
        emit(:set, :negate, text)
      else
        emit(:literal, :literal, text)
      end
    };

    '-' {
      text = copy(data, ts, te)
      # ranges cant start with a subset or intersection/negation/range operator
      if tokens.last[0] == :set
        emit(:literal, :literal, text)
      else
        emit(:set, :range, text)
      end
    };

    # Unlike ranges, intersections can start or end at set boundaries, whereupon
    # they match nothing: r = /[a&&]/; [r =~ ?a, r =~ ?&] # => [nil, nil]
    '&&' {
      emit(:set, :intersection, copy(data, ts, te))
    };

    backslash {
      fcall set_escape_sequence;
    };

    set_open >(open_bracket, 1) >set_opened {
      emit(:set, :open, copy(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error) {
      text = copy(data, ts, te)

      type = :posixclass
      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = class_name[1..-1]
        type = :nonposixclass
      end

      emit(type, class_name.to_sym, text)
    };

    # These are not supported in ruby at the moment. Enable them if they are.
    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error) {
    #   emit(:set, :collation, copy(data, ts, te))
    # };
    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error) {
    #   emit(:set, :equivalent, copy(data, ts, te))
    # };

    meta_char > (set_meta, 1) {
      emit(:literal, :literal, copy(data, ts, te))
    };

    any | ascii_nonprint | utf8_multibyte {
      text = copy(data, ts, te)
      emit(:literal, :literal, text)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    set_escape > (escaped_set_alpha, 2) {
      fhold;
      fnext character_set;
      fcall escape_sequence;
    };

    any > (escaped_set_alpha, 1) {
      emit(:escape, :literal, copy(data, ts-1, te))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = copy(data, ts-1, te)
      emit(:backref, :number, text)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, copy(data, ts-1, te))
      fret;
    };

    meta_char {
      case text = copy(data, ts-1, te)
      when '\.';  emit(:escape, :dot,               text)
      when '\|';  emit(:escape, :alternation,       text)
      when '\^';  emit(:escape, :bol,               text)
      when '\$';  emit(:escape, :eol,               text)
      when '\?';  emit(:escape, :zero_or_one,       text)
      when '\*';  emit(:escape, :zero_or_more,      text)
      when '\+';  emit(:escape, :one_or_more,       text)
      when '\(';  emit(:escape, :group_open,        text)
      when '\)';  emit(:escape, :group_close,       text)
      when '\{';  emit(:escape, :interval_open,     text)
      when '\}';  emit(:escape, :interval_close,    text)
      when '\[';  emit(:escape, :set_open,          text)
      when '\]';  emit(:escape, :set_close,         text)
      when "\\\\";
        emit(:escape, :backslash, text)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = copy(data, ts-1, te)
      when '\a'; emit(:escape, :bell,           text)
      when '\b'; emit(:escape, :backspace,      text)
      when '\e'; emit(:escape, :escape,         text)
      when '\f'; emit(:escape, :form_feed,      text)
      when '\n'; emit(:escape, :newline,        text)
      when '\r'; emit(:escape, :carriage,       text)
      when '\t'; emit(:escape, :tab,            text)
      when '\v'; emit(:escape, :vertical_tab,   text)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = copy(data, ts-1, te)
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text)
      else
        emit(:escape, :codepoint,      text)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) @eof(premature_end_error) {
      emit(:escape, :hex, copy(data, ts-1, te))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :control)
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :meta_sequence)
      fret;
    };

    char_type_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall char_type;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall unicode_property;
    };

    (any -- non_literal_escape) | utf8_multibyte > (escaped_alpha, 1) {
      emit(:escape, :literal, copy(data, ts-1, te))
      fret;
    };
  *|;

  # conditional expressions scanner
  # --------------------------------------------------------------------------
  conditional_expression := |*
    group_lookup . ')' {
      text = copy(data, ts, te-1)
      emit(:conditional, :condition, text)
      emit(:conditional, :condition_close, ')')
    };

    any {
      fhold;
      fcall main;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, copy(data, ts, te))
    };

    alternation {
      if conditional_stack.last == group_depth
        emit(:conditional, :separator, copy(data, ts, te))
      else
        emit(:meta, :alternation, copy(data, ts, te))
      end
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, copy(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, copy(data, ts, te))
    };

    backslash . keep_mark > (backslashed, 4) {
      emit(:keep, :mark, copy(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = copy(data, ts, te)
      when '\\A'; emit(:anchor, :bos,                text)
      when '\\z'; emit(:anchor, :eos,                text)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text)
      when '\\b'; emit(:anchor, :word_boundary,      text)
      when '\\B'; emit(:anchor, :nonword_boundary,   text)
      when '\\G'; emit(:anchor, :match_start,        text)
      end
    };

    literal_delimiters {
      append_literal(data, ts, te)
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open >set_opened {
      emit(:set, :open, copy(data, ts, te))
      fcall character_set;
    };

    # Conditional expression
    #   (?(condition)Y|N)   conditional expression
    # ------------------------------------------------------------------------
    conditional {
      text = copy(data, ts, te)

      conditional_stack << group_depth

      emit(:conditional, :open, text[0..-2])
      emit(:conditional, :condition_open, '(')
      fcall conditional_expression;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, copy(data, ts, te))
    };

    # Expression options:
    #   (?imxdau-imx)         option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #                         d: default class rules (1.9 compatible)
    #                         a: ASCII class rules (\s, \w, etc.)
    #                         u: Unicode class rules (\s, \w, etc.)
    #
    #   (?imxdau-imx:subexp)  option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      text = copy(data, ts, te)
      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
        raise InvalidGroupOption.new($1 || "-#{$2}", text)
      end
      emit_options(text)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = copy(data, ts, te)
      when '(?=';  emit(:assertion, :lookahead,    text)
      when '(?!';  emit(:assertion, :nlookahead,   text)
      when '(?<='; emit(:assertion, :lookbehind,   text)
      when '(?<!'; emit(:assertion, :nlookbehind,  text)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?~subexp)          absence group, matches anything that is not subexp
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = copy(data, ts, te)
      when '(?:';  emit(:group, :passive,      text)
      when '(?>';  emit(:group, :atomic,       text)
      when '(?~';  emit(:group, :absence,      text)

      when /^\(\?(?:<>|'')/
        validation_error(:group, 'named group', 'name is empty')

      when /^\(\?<[^>]+>/
        emit(:group, :named_ab,  text)

      when /^\(\?'[^']+'/
        emit(:group, :named_sq,  text)

      end
    };

    group_open @group_opened {
      text = copy(data, ts, te)
      emit(:group, :capture, text)
    };

    group_close @group_closed {
      if conditional_stack.last == group_depth + 1
        conditional_stack.pop
        emit(:conditional, :close, copy(data, ts, te))
      else
        if spacing_stack.length > 1 &&
           spacing_stack.last[:depth] == group_depth + 1
          spacing_stack.pop
          self.free_spacing = spacing_stack.last[:free_spacing]
        end

        emit(:group, :close, copy(data, ts, te))
      end
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_backref | group_number_backref) > (backslashed, 4) {
      case text = copy(data, ts, te)
      when /^\\k(<>|'')/
        validation_error(:backref, 'backreference', 'ref ID is empty')
      when /^\\k(.)[^\p{digit}\-][^+\-]*\D$/
        emit(:backref, $1 == '<' ? :name_ref_ab : :name_ref_sq, text)
      when /^\\k(.)\d+\D$/
        emit(:backref, $1 == '<' ? :number_ref_ab : :number_ref_sq, text)
      when /^\\k(.)-\d+\D$/
        emit(:backref, $1 == '<' ? :number_rel_ref_ab : :number_rel_ref_sq, text)
      when /^\\k(.)[^\p{digit}\-].*[+\-]\d+\D$/
        emit(:backref, $1 == '<' ? :name_recursion_ref_ab : :name_recursion_ref_sq, text)
      when /^\\k(.)-?\d+[+\-]\d+\D$/
        emit(:backref, $1 == '<' ? :number_recursion_ref_ab : :number_recursion_ref_sq, text)
      end
    };

    # Group call, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_call | group_number_call) > (backslashed, 4) {
      case text = copy(data, ts, te)
      when /^\\g(<>|'')/
        validation_error(:backref, 'subexpression call', 'ref ID is empty')
      when /^\\g(.)[^\p{digit}+\->][^+\-]*/
        emit(:backref, $1 == '<' ? :name_call_ab : :name_call_sq, text)
      when /^\\g(.)\d+\D$/
        emit(:backref, $1 == '<' ? :number_call_ab : :number_call_sq, text)
      when /^\\g(.)[+-]\d+/
        emit(:backref, $1 == '<' ? :number_rel_call_ab : :number_rel_call_sq, text)
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = copy(data, ts, te)
      when '?' ;  emit(:quantifier, :zero_or_one,            text)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text)
      end
    };

    zero_or_more {
      case text = copy(data, ts, te)
      when '*' ;  emit(:quantifier, :zero_or_more,            text)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text)
      end
    };

    one_or_more {
      case text = copy(data, ts, te)
      when '+' ;  emit(:quantifier, :one_or_more,            text)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text)
      when '++';  emit(:quantifier, :one_or_more_possessive, text)
      end
    };

    quantifier_interval  {
    quantifier_interval {
      emit(:quantifier, :interval, copy(data, ts, te))
    };

    # Catch unmatched curly braces as literals
    range_open {
      append_literal(data, ts, te)
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    comment {
      if free_spacing
        emit(:free_space, :comment, copy(data, ts, te))
      else
        # consume only the pound sign (#) and backtrack to do regular scanning
        append_literal(data, ts, ts + 1)
        fexec ts + 1;
      end
    };

    space+ {
      if free_spacing
        emit(:free_space, :whitespace, copy(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    (ascii_print -- space)+ | ascii_nonprint+ | utf8_multibyte+ {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from lib/regexp_parser/scanner/scanner.rl

require 'regexp_parser/error'

class Regexp::Scanner
  # General scanner error (catch all)
  class ScannerError < Regexp::Parser::Error; end

  # Base for all scanner validation errors
  class ValidationError < Regexp::Parser::Error
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid groupOption. Used for inline options.
  class InvalidGroupOption < ValidationError
    def initialize(option, text)
      super "Invalid group option #{option} in #{text}"
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input_object, options: nil, &block)
    new.scan(input_object, options: options, &block)
  end

  def scan(input_object, options: nil, &block)
    self.literal = nil
    stack = []

    input = input_object.is_a?(Regexp) ? input_object.source : input_object
    self.free_spacing = free_spacing?(input_object, options)
    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]

    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    self.tokens = []
    self.block  = block_given? ? block : nil

    self.set_depth = 0
    self.group_depth = 0
    self.conditional_stack = []
    self.char_pos = 0

    %% write data;
    %% write init;
    %% write exec;

    # to avoid "warning: assigned but unused variable - testEof"
    testEof = testEof

    if cs == re_scanner_error
      text = copy(data, ts ? ts-1 : 0, -1)
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{group_depth}]") if in_group?
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{set_depth}]") if in_set?

    # when the entire expression is a literal run
    emit_literal if literal

    tokens
  end

  # lazy-load property maps when first needed
  def self.short_prop_map
    @short_prop_map ||= parse_prop_map('short')
  end

  def self.long_prop_map
    @long_prop_map ||= parse_prop_map('long')
  end

  def self.parse_prop_map(name)
    File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
  end

  # Emits an array with the details of the scanned pattern
  def emit(type, token, text)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if literal

    # Ragel runs with byte-based indices (ts, te). These are of little value to
    # end-users, so we keep track of char-based indices and emit those instead.
    ts_char_pos = char_pos
    te_char_pos = char_pos + text.length

    if block
      block.call type, token, text, ts_char_pos, te_char_pos
    end

    tokens << [type, token, text, ts_char_pos, te_char_pos]

    self.char_pos = te_char_pos
  end

  private

  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                :group_depth, :set_depth, :conditional_stack, :char_pos

  def free_spacing?(input_object, options)
    if options && !input_object.is_a?(String)
      raise ArgumentError, 'options cannot be supplied unless scanning a String'
    end

    options = input_object.options if input_object.is_a?(::Regexp)

    return false unless options

    options & Regexp::EXTENDED != 0
  end

  def in_group?
    group_depth > 0
  end

  def in_set?
    set_depth > 0
  end

  # Copy from ts to te from data as text
  def copy(data, ts, te)
    data[ts...te].pack('c*').force_encoding('utf-8')
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal.
  def append_literal(data, ts, te)
    self.literal = literal || []
    literal << copy(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method.
  def emit_literal
    text = literal.join
    self.literal = nil
    emit(:literal, :literal, text)
  end

  def emit_options(text)
    token = nil

    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
    positive, negative, group_local = $1, $2, $3

    if positive.include?('x')
      self.free_spacing = true
    end

    # If the x appears in both, treat it like ruby does, the second cancels
    # the first.
    if negative && negative.include?('x')
      self.free_spacing = false
    end

    if group_local
      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
      token = :options
    else
      # switch for parent group level
      spacing_stack.last[:free_spacing] = free_spacing
      token = :options_switch
    end

    emit(:group, token, text)
  end

  def emit_meta_control_sequence(data, ts, te, token)
    if data.last < 0x00 || data.last > 0x7F
      validation_error(:sequence, 'escape', token.to_s)
    end
    emit(:escape, token, copy(data, ts-1, te))
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    end

    raise error # unless @@config.validation_ignore
  end
end # module Regexp::Scanner

%%{
  machine re_scanner;
  include re_char_type "char_type.rl";
  include re_property  "property.rl";

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
  utf8_multibyte        = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parentheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  comment               = ('#' . [^\n]* . '\n'?);

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby at the moment
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrtv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;

  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;

  sequence_char         = [CMcux];

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantity_exact        = (digit+);
  quantity_minimum      = (digit+) . ',';
  quantity_maximum      = ',' . (digit+);
  quantity_range        = (digit+) . ',' . (digit+);
  quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
                          quantity_maximum | quantity_range ) . range_close .
                          quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  conditional           = '(?(';

  group_comment         = '?#' . [^)]* . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_absence         = '?~';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  # try to treat every other group head as options group, like Ruby
  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;

  group_ref             = [gk];
  group_name_id_ab      = ([^0-9\->] | utf8_multibyte) . ([^>] | utf8_multibyte)*;
  group_name_id_sq      = ([^0-9\-'] | utf8_multibyte) . ([^'] | utf8_multibyte)*;
  group_number          = '-'? . [1-9] . [0-9]*;
  group_level           = [+\-] . [0-9]+;

  group_name            = ('<' . group_name_id_ab? . '>') |
                          ("'" . group_name_id_sq? . "'");
  group_lookup          = group_name | group_number;

  group_named           = ('?' . group_name );

  group_name_backref    = 'k' . (('<' . group_name_id_ab? . group_level? '>') |
                                 ("'" . group_name_id_sq? . group_level? "'"));
  group_name_call       = 'g' . (('<' . group_name_id_ab? . group_level? '>') |
                                 ("'" . group_name_id_sq? . group_level? "'"));

  group_number_backref  = 'k' . (('<' . group_number . group_level? '>') |
                                 ("'" . group_number . group_level? "'"));
  group_number_call     = 'g' . (('<' . ((group_number . group_level?) | '0') '>') |
                                 ("'" . ((group_number . group_level?) | '0') "'"));

  group_type            = group_atomic | group_passive | group_absence | group_named;

  keep_mark             = 'K';

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parentheses | brackets |
                          line_anchor | quantifier_greedy;

  literal_delimiters    = ']' | '}';

  ascii_print           = ((0x20..0x7e) - meta_char - '#');
  ascii_nonprint        = (0x01..0x1f | 0x7f);

  non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
                          keep_mark | sequence_char;

  # escapes that also work within a character set
  set_escape            = backslash | brackets | escaped_ascii | property_char |
                          sequence_char | single_codepoint_char_type;

  # EOF error, used where it can be detected
  action premature_end_error {
    text = copy(data, ts ? ts-1 : 0, -1)
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = copy(data, ts ? ts-1 : 0, -1)
    validation_error(:sequence, 'sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { self.group_depth = group_depth + 1 }
  action group_closed { self.group_depth = group_depth - 1 }
  action set_opened   { self.set_depth   = set_depth   + 1 }
  action set_closed   { self.set_depth   = set_depth   - 1 }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    set_close > (set_meta, 2) @set_closed {
      emit(:set, :close, copy(data, ts, te))
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-]' @set_closed { # special case, emits two tokens
      emit(:literal, :literal, copy(data, ts, te-1))
      emit(:set, :close, copy(data, ts+1, te))
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-&&' { # special case, emits two tokens
      emit(:literal, :literal, '-')
      emit(:set, :intersection, '&&')
    };

    '^' {
      text = copy(data, ts, te)
      if tokens.last[1] == :open
        emit(:set, :negate, text)
      else
        emit(:literal, :literal, text)
      end
    };

    '-' {
      text = copy(data, ts, te)
      # ranges cant start with a subset or intersection/negation/range operator
      if tokens.last[0] == :set
        emit(:literal, :literal, text)
      else
        emit(:set, :range, text)
      end
    };

    # Unlike ranges, intersections can start or end at set boundaries, whereupon
    # they match nothing: r = /[a&&]/; [r =~ ?a, r =~ ?&] # => [nil, nil]
    '&&' {
      emit(:set, :intersection, copy(data, ts, te))
    };

    backslash {
      fcall set_escape_sequence;
    };

    set_open >(open_bracket, 1) >set_opened {
      emit(:set, :open, copy(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
      text = copy(data, ts, te)

      type = :posixclass
      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = class_name[1..-1]
        type = :nonposixclass
      end

      emit(type, class_name.to_sym, text)
    };

    # These are not supported in ruby at the moment. Enable them if they are.
    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
    #   emit(:set, :collation, copy(data, ts, te))
    # };
    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
    #   emit(:set, :equivalent, copy(data, ts, te))
    # };

    meta_char > (set_meta, 1) {
      emit(:literal, :literal, copy(data, ts, te))
    };

    any | ascii_nonprint | utf8_multibyte {
      text = copy(data, ts, te)
      emit(:literal, :literal, text)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    set_escape > (escaped_set_alpha, 2) {
      fhold;
      fnext character_set;
      fcall escape_sequence;
    };

    any > (escaped_set_alpha, 1) {
      emit(:escape, :literal, copy(data, ts-1, te))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = copy(data, ts-1, te)
      emit(:backref, :number, text)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, copy(data, ts-1, te))
      fret;
    };

    meta_char {
      case text = copy(data, ts-1, te)
      when '\.';  emit(:escape, :dot,               text)
      when '\|';  emit(:escape, :alternation,       text)
      when '\^';  emit(:escape, :bol,               text)
      when '\$';  emit(:escape, :eol,               text)
      when '\?';  emit(:escape, :zero_or_one,       text)
      when '\*';  emit(:escape, :zero_or_more,      text)
      when '\+';  emit(:escape, :one_or_more,       text)
      when '\(';  emit(:escape, :group_open,        text)
      when '\)';  emit(:escape, :group_close,       text)
      when '\{';  emit(:escape, :interval_open,     text)
      when '\}';  emit(:escape, :interval_close,    text)
      when '\[';  emit(:escape, :set_open,          text)
      when '\]';  emit(:escape, :set_close,         text)
      when "\\\\";
        emit(:escape, :backslash, text)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = copy(data, ts-1, te)
      when '\a'; emit(:escape, :bell,           text)
      when '\b'; emit(:escape, :backspace,      text)
      when '\e'; emit(:escape, :escape,         text)
      when '\f'; emit(:escape, :form_feed,      text)
      when '\n'; emit(:escape, :newline,        text)
      when '\r'; emit(:escape, :carriage,       text)
      when '\t'; emit(:escape, :tab,            text)
      when '\v'; emit(:escape, :vertical_tab,   text)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = copy(data, ts-1, te)
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text)
      else
        emit(:escape, :codepoint,      text)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) @eof(premature_end_error) {
      emit(:escape, :hex, copy(data, ts-1, te))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :control)
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :meta_sequence)
      fret;
    };

    char_type_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall char_type;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall unicode_property;
    };

    (any -- non_literal_escape) | utf8_multibyte > (escaped_alpha, 1) {
      emit(:escape, :literal, copy(data, ts-1, te))
      fret;
    };
  *|;

  # conditional expressions scanner
  # --------------------------------------------------------------------------
  conditional_expression := |*
    group_lookup . ')' {
      text = copy(data, ts, te-1)
      emit(:conditional, :condition, text)
      emit(:conditional, :condition_close, ')')
    };

    any {
      fhold;
      fcall main;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, copy(data, ts, te))
    };

    alternation {
      if conditional_stack.last == group_depth
        emit(:conditional, :separator, copy(data, ts, te))
      else
        emit(:meta, :alternation, copy(data, ts, te))
      end
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, copy(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, copy(data, ts, te))
    };

    backslash . keep_mark > (backslashed, 4) {
      emit(:keep, :mark, copy(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = copy(data, ts, te)
      when '\\A'; emit(:anchor, :bos,                text)
      when '\\z'; emit(:anchor, :eos,                text)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text)
      when '\\b'; emit(:anchor, :word_boundary,      text)
      when '\\B'; emit(:anchor, :nonword_boundary,   text)
      when '\\G'; emit(:anchor, :match_start,        text)
      end
    };

    literal_delimiters {
      append_literal(data, ts, te)
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open >set_opened {
      emit(:set, :open, copy(data, ts, te))
      fcall character_set;
    };

    # Conditional expression
    #   (?(condition)Y|N)   conditional expression
    # ------------------------------------------------------------------------
    conditional {
      text = copy(data, ts, te)

      conditional_stack << group_depth

      emit(:conditional, :open, text[0..-2])
      emit(:conditional, :condition_open, '(')
      fcall conditional_expression;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, copy(data, ts, te))
    };

    # Expression options:
    #   (?imxdau-imx)         option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #                         d: default class rules (1.9 compatible)
    #                         a: ASCII class rules (\s, \w, etc.)
    #                         u: Unicode class rules (\s, \w, etc.)
    #
    #   (?imxdau-imx:subexp)  option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      text = copy(data, ts, te)
      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
        raise InvalidGroupOption.new($1 || "-#{$2}", text)
      end
      emit_options(text)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = copy(data, ts, te)
      when '(?=';  emit(:assertion, :lookahead,    text)
      when '(?!';  emit(:assertion, :nlookahead,   text)
      when '(?<='; emit(:assertion, :lookbehind,   text)
      when '(?<!'; emit(:assertion, :nlookbehind,  text)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?~subexp)          absence group, matches anything that is not subexp
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = copy(data, ts, te)
      when '(?:';  emit(:group, :passive,      text)
      when '(?>';  emit(:group, :atomic,       text)
      when '(?~';  emit(:group, :absence,      text)

      when /^\(\?(?:<>|'')/
        validation_error(:group, 'named group', 'name is empty')

      when /^\(\?<[^>]+>/
        emit(:group, :named_ab,  text)

      when /^\(\?'[^']+'/
        emit(:group, :named_sq,  text)

      end
    };

    group_open @group_opened {
      text = copy(data, ts, te)
      emit(:group, :capture, text)
    };

    group_close @group_closed {
      if conditional_stack.last == group_depth + 1
        conditional_stack.pop
        emit(:conditional, :close, copy(data, ts, te))
      else
        if spacing_stack.length > 1 &&
           spacing_stack.last[:depth] == group_depth + 1
          spacing_stack.pop
          self.free_spacing = spacing_stack.last[:free_spacing]
        end

        emit(:group, :close, copy(data, ts, te))
      end
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_backref | group_number_backref) > (backslashed, 4) {
      case text = copy(data, ts, te)
      when /^\\k(<>|'')/
        validation_error(:backref, 'backreference', 'ref ID is empty')
      when /^\\k(.)[^\p{digit}\-][^+\-]*\D$/
        emit(:backref, $1 == '<' ? :name_ref_ab : :name_ref_sq, text)
      when /^\\k(.)\d+\D$/
        emit(:backref, $1 == '<' ? :number_ref_ab : :number_ref_sq, text)
      when /^\\k(.)-\d+\D$/
        emit(:backref, $1 == '<' ? :number_rel_ref_ab : :number_rel_ref_sq, text)
      when /^\\k(.)[^\p{digit}\-].*[+\-]\d+\D$/
        emit(:backref, $1 == '<' ? :name_recursion_ref_ab : :name_recursion_ref_sq, text)
      when /^\\k(.)-?\d+[+\-]\d+\D$/
        emit(:backref, $1 == '<' ? :number_recursion_ref_ab : :number_recursion_ref_sq, text)
      end
    };

    # Group call, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_call | group_number_call) > (backslashed, 4) {
      case text = copy(data, ts, te)
      when /^\\g(<>|'')/
        validation_error(:backref, 'subexpression call', 'ref ID is empty')
      when /^\\g(.)[^\p{digit}+\->][^+\-]*/
        emit(:backref, $1 == '<' ? :name_call_ab : :name_call_sq, text)
      when /^\\g(.)\d+\D$/
        emit(:backref, $1 == '<' ? :number_call_ab : :number_call_sq, text)
      when /^\\g(.)[+-]\d+/
        emit(:backref, $1 == '<' ? :number_rel_call_ab : :number_rel_call_sq, text)
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = copy(data, ts, te)
      when '?' ;  emit(:quantifier, :zero_or_one,            text)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text)
      end
    };

    zero_or_more {
      case text = copy(data, ts, te)
      when '*' ;  emit(:quantifier, :zero_or_more,            text)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text)
      end
    };

    one_or_more {
      case text = copy(data, ts, te)
      when '+' ;  emit(:quantifier, :one_or_more,            text)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text)
      when '++';  emit(:quantifier, :one_or_more_possessive, text)
      end
    };

    quantifier_interval  {
      emit(:quantifier, :interval, copy(data, ts, te))
    };

    # Catch unmatched curly braces as literals
    range_open {
      append_literal(data, ts, te)
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    comment {
      if free_spacing
        emit(:free_space, :comment, copy(data, ts, te))
      else
        # consume only the pound sign (#) and backtrack to do regular scanning
        append_literal(data, ts, ts + 1)
        fexec ts + 1;
      end
    };

    space+ {
      if free_spacing
        emit(:free_space, :whitespace, copy(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    (ascii_print -- space)+ | ascii_nonprint+ | utf8_multibyte+ {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from lib/regexp_parser/scanner/scanner.rl

require 'regexp_parser/error'

class Regexp::Scanner
  # General scanner error (catch all)
  class ScannerError < Regexp::Parser::Error; end

  # Base for all scanner validation errors
  class ValidationError < Regexp::Parser::Error
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid groupOption. Used for inline options.
  class InvalidGroupOption < ValidationError
    def initialize(option, text)
      super "Invalid group option #{option} in #{text}"
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input_object, options: nil, &block)
    new.scan(input_object, options: options, &block)
  end

  def scan(input_object, options: nil, &block)
    self.literal = nil
    stack = []

    input = input_object.is_a?(Regexp) ? input_object.source : input_object
    self.free_spacing = free_spacing?(input_object, options)
    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]

    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    self.tokens = []
    self.block  = block_given? ? block : nil

    self.set_depth = 0
    self.group_depth = 0
    self.conditional_stack = []
    self.char_pos = 0

    %% write data;
    %% write init;
    %% write exec;

    # to avoid "warning: assigned but unused variable - testEof"
    testEof = testEof

    if cs == re_scanner_error
      text = copy(data, ts ? ts-1 : 0, -1)
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{group_depth}]") if in_group?
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{set_depth}]") if in_set?

    # when the entire expression is a literal run
    emit_literal if literal

    tokens
  end

  # lazy-load property maps when first needed
  def self.short_prop_map
    @short_prop_map ||= parse_prop_map('short')
  end

  def self.long_prop_map
    @long_prop_map ||= parse_prop_map('long')
  end

  def self.parse_prop_map(name)
    File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
  end

  # Emits an array with the details of the scanned pattern
  def emit(type, token, text)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if literal

    # Ragel runs with byte-based indices (ts, te). These are of little value to
    # end-users, so we keep track of char-based indices and emit those instead.
    ts_char_pos = char_pos
    te_char_pos = char_pos + text.length

    if block
      block.call type, token, text, ts_char_pos, te_char_pos
    end

    tokens << [type, token, text, ts_char_pos, te_char_pos]

    self.char_pos = te_char_pos
  end

  private

  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                :group_depth, :set_depth, :conditional_stack, :char_pos

  def free_spacing?(input_object, options)
    if options && !input_object.is_a?(String)
      raise ArgumentError, 'options cannot be supplied unless scanning a String'
    end

    options = input_object.options if input_object.is_a?(::Regexp)

    return false unless options

    options & Regexp::EXTENDED != 0
  end

  def in_group?
    group_depth > 0
  end

  def in_set?
    set_depth > 0
  end

  # Copy from ts to te from data as text
  def copy(data, ts, te)
    data[ts...te].pack('c*').force_encoding('utf-8')
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal.
  def append_literal(data, ts, te)
    self.literal = literal || []
    literal << copy(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method.
  def emit_literal
    text = literal.join
    self.literal = nil
    emit(:literal, :literal, text)
  end

  def emit_options(text)
    token = nil

    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
    positive, negative, group_local = $1, $2, $3

    if positive.include?('x')
      self.free_spacing = true
    end

    # If the x appears in both, treat it like ruby does, the second cancels
    # the first.
    if negative && negative.include?('x')
      self.free_spacing = false
    end

    if group_local
      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
      token = :options
    else
      # switch for parent group level
      spacing_stack.last[:free_spacing] = free_spacing
      token = :options_switch
    end

    emit(:group, token, text)
  end

  def emit_meta_control_sequence(data, ts, te, token)
    if data.last < 0x00 || data.last > 0x7F
      validation_error(:sequence, 'escape', token.to_s)
    end
    emit(:escape, token, copy(data, ts-1, te))
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    end

    raise error # unless @@config.validation_ignore
  end
end # module Regexp::Scanner


Don't use define_method in the lexer.

Don't use define_method in the lexer.

Profiling showed that calls to methods defined using `define_method` are
really, really slow. Before this commit the lexer would process 3000-4000
lines per second. With this commit that has been increased to around 10 000
lines per second.

Thanks to @headius for mentioning the (potential) overhead of define_method.

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::XML::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    # @!attribute [r] tokens
    #  @return [Array]
    #
    class Lexer
      %% write data;

      # % fix highlight

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Set]
      #
      HTML_VOID_ELEMENTS = Set.new([
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ])

      ##
      # @param [String] data The data to lex.
      #
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(data, options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        @data = data

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @ts       = nil
        @te       = nil
        @stack    = []
        @top      = 0
        @cs       = self.class.lexer_start
        @act      = 0
        @elements = []
        @eof      = @data.bytesize
        @p        = 0
        @pe       = @eof

        @buffer_start_position = nil
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # This method resets the internal state of the lexer after consuming the
      # input.
      #
      # @param [String] data The string to consume.
      # @return [Array]
      # @see #advance
      #
      def lex
        tokens = []

        advance do |token|
          tokens << token
        end

        reset

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # This method does *not* reset the internal state of the lexer.
      #
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        _lexer_eof_trans          = self.class.send(:_lexer_eof_trans)
        _lexer_from_state_actions = self.class.send(:_lexer_from_state_actions)
        _lexer_index_offsets      = self.class.send(:_lexer_index_offsets)
        _lexer_indicies           = self.class.send(:_lexer_indicies)
        _lexer_key_spans          = self.class.send(:_lexer_key_spans)
        _lexer_to_state_actions   = self.class.send(:_lexer_to_state_actions)
        _lexer_trans_actions      = self.class.send(:_lexer_trans_actions)
        _lexer_trans_keys         = self.class.send(:_lexer_trans_keys)
        _lexer_trans_targs        = self.class.send(:_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start = @ts, stop = @te)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # By default `@ts` and `@te` are used as the start/stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start = @ts, stop = @te)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @block.call(token)
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer, set to `@te`
      #  by default.
      #
      def start_buffer(position = @te)
        @buffer_start_position = position
      end

      ##
      # Returns `true` if we're currently buffering.
      #
      # @return [TrueClass|FalseClass]
      #
      def buffering?
        return !!@buffer_start_position
      end

      ##
      # Emits the current buffer if we have any. The current line number is
      # advanced based on the amount of newlines in the buffer.
      #
      # @param [Fixnum] position The end position of the buffer, set to `@ts`
      #  by default.
      #
      # @param [Symbol] type The type of node to emit.
      #
      def emit_buffer(position = @ts, type = :T_TEXT)
        return unless @buffer_start_position

        content = text(@buffer_start_position, position)

        unless content.empty?
          add_token(type, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end

        @buffer_start_position = nil
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        # Use instance variables for `ts` and friends.
        access @;
        getkey (@data.getbyte(@p) || 0);
        variable p @p;
        variable pe @pe;
        variable eof @eof;

        newline    = '\n' | '\r\n';
        whitespace = [ \t];
        identifier = [a-zA-Z0-9\-_:]+;

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        action start_string_dquote {
          start_buffer

          fcall string_dquote;
        }

        action start_string_squote {
          start_buffer

          fcall string_squote;
        }

        # Machine for processing double quoted strings.
        string_dquote := |*
          dquote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          squote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+;

        action start_doctype {
          emit_buffer
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE) };

          # Lex the public/system IDs as regular strings.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          identifier => { emit(:T_DOCTYPE_NAME) };

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          emit_buffer
          add_token(:T_CDATA_START)

          start_buffer

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          cdata_end => {
            emit_buffer
            add_token(:T_CDATA_END)

            fret;
          };

          any;
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          emit_buffer
          add_token(:T_COMMENT_START)

          start_buffer

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          comment_end => {
            emit_buffer
            add_token(:T_COMMENT_END)

            fret;
          };

          any;
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          emit_buffer
          add_token(:T_XML_DECL_START)

          start_buffer

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            emit_buffer
            add_token(:T_XML_DECL_END)

            fret;
          };

          # Attributes and their values (e.g. version="1.0").
          identifier => { emit(:T_ATTR) };

          dquote => start_string_dquote;
          squote => start_string_squote;

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          emit_buffer
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(@ts + 1)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name if html

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_start = '<' identifier;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          identifier => { emit(:T_ATTR) };

          # Attribute values.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' identifier '>' => {
            emit_buffer
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          any => {
            # First character, start buffering (unless we already are buffering).
            start_buffer(@ts) unless buffering?

            # EOF, emit the text buffer.
            if @te == @eof
              emit_buffer(@te)
            end
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::XML::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    # @!attribute [r] tokens
    #  @return [Array]
    #
    class Lexer
      %% write data;

      # % fix highlight

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Set]
      #
      HTML_VOID_ELEMENTS = Set.new([
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ])

      # Lazy way of forwarding instance method calls used internally by Ragel
      # to their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      ##
      # @param [String] data The data to lex.
      #
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(data, options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        @data = data

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @ts       = nil
        @te       = nil
        @stack    = []
        @top      = 0
        @cs       = self.class.lexer_start
        @act      = 0
        @elements = []
        @eof      = @data.bytesize
        @p        = 0
        @pe       = @eof

        @buffer_start_position = nil
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # This method resets the internal state of the lexer after consuming the
      # input.
      #
      # @param [String] data The string to consume.
      # @return [Array]
      # @see #advance
      #
      def lex
        tokens = []

        advance do |token|
          tokens << token
        end

        reset

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # This method does *not* reset the internal state of the lexer.
      #
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start = @ts, stop = @te)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # By default `@ts` and `@te` are used as the start/stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start = @ts, stop = @te)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @block.call(token)
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer, set to `@te`
      #  by default.
      #
      def start_buffer(position = @te)
        @buffer_start_position = position
      end

      ##
      # Returns `true` if we're currently buffering.
      #
      # @return [TrueClass|FalseClass]
      #
      def buffering?
        return !!@buffer_start_position
      end

      ##
      # Emits the current buffer if we have any. The current line number is
      # advanced based on the amount of newlines in the buffer.
      #
      # @param [Fixnum] position The end position of the buffer, set to `@ts`
      #  by default.
      #
      # @param [Symbol] type The type of node to emit.
      #
      def emit_buffer(position = @ts, type = :T_TEXT)
        return unless @buffer_start_position

        content = text(@buffer_start_position, position)

        unless content.empty?
          add_token(type, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end

        @buffer_start_position = nil
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        # Use instance variables for `ts` and friends.
        access @;
        getkey (@data.getbyte(@p) || 0);
        variable p @p;
        variable pe @pe;
        variable eof @eof;

        newline    = '\n' | '\r\n';
        whitespace = [ \t];
        identifier = [a-zA-Z0-9\-_:]+;

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        action start_string_dquote {
          start_buffer

          fcall string_dquote;
        }

        action start_string_squote {
          start_buffer

          fcall string_squote;
        }

        # Machine for processing double quoted strings.
        string_dquote := |*
          dquote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          squote => {
            emit_buffer(@ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+;

        action start_doctype {
          emit_buffer
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE) };

          # Lex the public/system IDs as regular strings.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          identifier => { emit(:T_DOCTYPE_NAME) };

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          emit_buffer
          add_token(:T_CDATA_START)

          start_buffer

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          cdata_end => {
            emit_buffer
            add_token(:T_CDATA_END)

            fret;
          };

          any;
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          emit_buffer
          add_token(:T_COMMENT_START)

          start_buffer

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          comment_end => {
            emit_buffer
            add_token(:T_COMMENT_END)

            fret;
          };

          any;
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          emit_buffer
          add_token(:T_XML_DECL_START)

          start_buffer

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            emit_buffer
            add_token(:T_XML_DECL_END)

            fret;
          };

          # Attributes and their values (e.g. version="1.0").
          identifier => { emit(:T_ATTR) };

          dquote => start_string_dquote;
          squote => start_string_squote;

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          emit_buffer
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(@ts + 1)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name if html

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_start = '<' identifier;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          identifier => { emit(:T_ATTR) };

          # Attribute values.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' identifier '>' => {
            emit_buffer
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          any => {
            # First character, start buffering (unless we already are buffering).
            start_buffer(@ts) unless buffering?

            # EOF, emit the text buffer.
            if @te == @eof
              emit_buffer(@te)
            end
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga


Move from scanner-type lexer to match-based

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' ( any -- '"' | '\\"' )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  body = ( >mark number        %{ emit :number     } ) |
         ( >mark string_double %{ emit :dstring    } ) |
         ( >mark string_single %{ emit :sstring    } ) |
         (       'class'       %{ emit :class      } ) |
         (       'module'      %{ emit :module     } ) |
         (       'if'          %{ emit :if         } ) |
         (       'unless'      %{ emit :unless     } ) |
         (       'elsif'       %{ emit :elsif      } ) |
         (       'else'        %{ emit :else       } ) |
         ( >mark unops         %{ emit :unop       } ) |
         ( >mark binops        %{ emit :binop      } ) |
         ( >mark keywords      %{ emit :keyword    } ) |
         ( >mark identifier    %{ emit :identifier } ) |
         (       '->'          %{ emit :arrow      } ) |
         (       '='           %{ emit :equal      } ) |
         (       '{'           %{ emit :lbrack     } ) |
         (       '('           %{ emit :lparen     } ) |
         (       '['           %{ emit :lbrace     } ) |
         (       '}'           %{ emit :rbrack     } ) |
         (       ')'           %{ emit :rparen     } ) |
         (       ']'           %{ emit :rbrace     } ) |
         (       ':'           %{ emit :colon      } ) |
         (       '.'           %{ emit :prop       } ) |
         (       ','           %{ emit :comma      } ) |
         (       '\n'          %{ line.call        } )
  ;

  main := body**;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type, cur = nil)
        @tokens << if cur
          Token.new(type, @data[@start..(@cur - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@ts..(@te-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' ( any -- '"' | '\\"' )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  main := |*
    number        => { emit :number      };
    string_double => { emit :dstring     };
    string_single => { emit :sstring     };
    'class'       => { emit :class       };
    'module'      => { emit :module      };
    'if'          => { emit :if          };
    'unless'      => { emit :unless      };
    'elsif'       => { emit :elsif       };
    'else'        => { emit :else        };
    unops         => { emit :unop        };
    binops        => { emit :binop       };
    keywords      => { emit :keyword     };
    identifier    => { emit :identifier  };
    '->'          => { emit :arrow       };
    '='           => { emit :equal       };
    '{'           => { emit :lbrack      };
    '('           => { emit :lparen      };
    '['           => { emit :lbrace      };
    '}'           => { emit :rbrack      };
    ')'           => { emit :rparen      };
    ']'           => { emit :rbrace      };
    ':'           => { emit :colon       };
    '.'           => { emit :prop        };
    ','           => { emit :comma       };
    '\n'          => { line.call         };
    space         => {                   };
    any           => { error             };
  *|;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
      end

      def emit(type)
        @tokens << Token.new(type, @data[@ts..(@te - 1)],
          @line[:num], @ts - @line[:start])
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@ts..(@te-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end


Small line wrapping change in the lexer.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
      @stack  = []
      @top    = 0

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length) if value

      @tokens << token
    end

    def emit_text_buffer
      return if @text_buffer.empty?

      add_token(:T_TEXT, @text_buffer)

      @text_buffer = ''
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      action emit_newline {
        t(:T_TEXT)
        advance_line
      }

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        fcall string_dquote;
      }

      action string_squote {
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer
          t(:T_CDATA_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      comment := |*
        comment_end => {
          emit_text_buffer
          t(:T_COMMENT_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action open_element {
        add_token(:T_ELEM_OPEN, nil)
        advance_column

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)

          # Advance the column for the colon (:) that separates the namespace
          # and element name.
          advance_column
        end

        add_token(:T_ELEM_NAME, name)

        fcall element;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      element_text := |*
        ^'<' => buffer_text;

        '<' => {
          emit_text_buffer
          fhold;
          fret;
        };
      *|;

      element_closing_tag := |*
        whitespace => { advance_column };

        element_name => {
          emit_text_buffer
          add_token(:T_ELEM_CLOSE, nil)

          # Advance the column for the </
          advance_column(2)

          # Advance the column for the closing name.
          advance_column(@te - p)
          fret;
        };

        '>' => { fret; };
      *|;

      element := |*
        whitespace => { advance_column };

        element_start => open_element;

        # Consume the text inside the element.
        '>' => {
          advance_column
          fcall element_text;
        };

        # Attributes and their values.
        element_name %{ t(:T_ATTR, @ts, p) }

        # The value of the attribute. Attribute values are not required. e.g.
        # in <p data-foo></p> data-foo would be a boolean attribute.
        (
          '=' >{ advance_column }

          # The value of the attribute, wrapped in either single or double
          # quotes.
          (dquote @string_dquote | squote @string_squote)
        )*;

        # Non self-closing elements.
        '</' => {
          fcall element_closing_tag;
          fret;
        };

        # self-closing / void elements.
        '/>' => {
          advance_column
          add_token(:T_ELEM_CLOSE, nil)
          fret;
        };
      *|;

      main := |*
        newline => emit_newline;

        doctype_start => {
          t(:T_DOCTYPE_START)
          fcall doctype;
        };

        cdata_start => {
          t(:T_CDATA_START)
          fcall cdata;
        };

        comment_start => {
          t(:T_COMMENT_START)
          fcall comment;
        };

        element_start => open_element;
      *|;
    }%%
  end # Lexer
end # Oga