Specified exact path of ragel file in generated scanner output

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  hex_or_not            = (xdigit|[^0-9a-fA-F}]); # note closing curly at end

  wide_hex_seq_invalid  = 'x' . '{' . hex_or_not{1,9};
  wide_hex_seq_empty    = 'x' . '{' . (space+)? . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-');

  meta_sequence         = 'M-' . (backslash . control_sequence)?;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . [\-mix];

  group_ref             = [gk];
  group_name            = (alnum . (alnum+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");

  group_name_ref        = group_ref . (('<' . group_name . group_level? '>') |
                                       ("'" . group_name . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise InvalidSequenceError.new('sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { group_depth += 1; in_group = true }
  action group_closed { group_depth -= 1; in_group = group_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :close, *text(data, ts, te))

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-]' { # special case, emits two tokens
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :member, copy(data, ts..te-2), ts, te)
      emit(set_type, :close,  copy(data, ts+1..te-1), ts, te)

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '^' {
      text = text(data, ts, te).first
      if @tokens.last[1] == :open
        emit(set_type, :negate, text, ts, te)
      else
        emit(set_type, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum {
      emit(set_type, :range, *text(data, ts, te))
    };

    '&&' {
      emit(set_type, :intersection, *text(data, ts, te))
    };

    '\\' {
      fcall set_escape_sequence;
    };

    '[' >(open_bracket, 1) {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @eof(premature_end_error) {
      text = text(data, ts, te).first

      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = "non#{class_name[1..-1]}"
      end

      token_sym = "class_#{class_name}".to_sym
      emit(set_type, token_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :equivalent, *text(data, ts, te))
    };

    # exclude the closing bracket as a cleaner workaround for dealing with the
    # ambiguity caused upon exit from the unicode properties machine
    meta_char -- ']' {
      emit(set_type, :member, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      emit(set_type, :member, *text(data, ts, te))
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' > (escaped_set_alpha, 2) {
      emit(set_type, :backspace, *text(data, ts, te, 1))
      fret;
    };

    char_type > (escaped_set_alpha, 4) {
      case text = text(data, ts, te, 1).first
      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      emit(set_type, :range_hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence {
      emit(set_type, :member_hex, *text(data, ts, te, 1))
      fret;
    };

    meta_char | [\\\]\-\,] {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };

    property_char > (escaped_set_alpha, 3) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; emit(:escape, :space,          text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex_wide, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    (wide_hex_seq_invalid | wide_hex_seq_empty) {
      raise InvalidSequenceError.new("wide hex sequence")
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("control sequence")
        end
      else
        raise PrematureEndError.new("control sequence")
      end
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("meta sequence")
        end
      else
        raise PrematureEndError.new("meta sequence")
      end
      fret;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext main;
      fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      emit(:meta, :alternation, *text(data, ts, te))
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in anchor at #{text} (char #{ts})")
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = text(data, ts, te).first
      when '\\d'; emit(:type, :digit,      text, ts, te)
      when '\\D'; emit(:type, :nondigit,   text, ts, te)
      when '\\h'; emit(:type, :hex,        text, ts, te)
      when '\\H'; emit(:type, :nonhex,     text, ts, te)
      when '\\s'; emit(:type, :space,      text, ts, te)
      when '\\S'; emit(:type, :nonspace,   text, ts, te)
      when '\\w'; emit(:type, :word,       text, ts, te)
      when '\\W'; emit(:type, :nonword,    text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in type at #{text} (char #{ts})")
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      p = scan_options(p, data, ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)

      when /^\(\?<(\w*)>/
        empty_name_error(:group, 'named group (ab)') if $1.empty?

        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'(\w*)'/
        empty_name_error(:group, 'named group (sq)') if $1.empty?

        emit(:group, :named_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown subexpression group format '#{text}'")
      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      emit(:group, :close, *text(data, ts, te))
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])<>/ # angle brackets
        empty_backref_error("ref/call (ab)")

      when /^\\([gk])''/ # single quotes
        empty_backref_error("ref/call (sq)")

      when /^\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab,  text, ts, te)
        else
          emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /^\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq,  text, ts, te)
        else
          emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /^\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /^\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_rel_call_sq,  text, ts, te)
        end

      when /^\\k<[^\d-](\w+)?[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_nest_ref_ab,  text, ts, te)

      when /^\\k'[^\d-](\w+)?[+\-]\d+'/ # single-quotes
        emit(:backref, :name_nest_ref_sq,  text, ts, te)

      when /^\\([gk])<\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_nest_ref_ab,  text, ts, te)

      when /^\\([gk])'\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_nest_ref_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown backreference format '#{text}'")
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from lib/regexp_parser/scanner/scanner.rl

module Regexp::Scanner
  %% write data;

  # General scanner error (catch all)
  class ScannerError < StandardError
    def initialize(what)
      super what
    end
  end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group, group_depth = false, 0
    in_set,   set_depth, set_type   = false, 0, :set

    %% write init;
    %% write exec;

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{in_group}:#{group_depth}]") if in_group
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{in_set}:#{set_depth}]") if in_set

    # when the entire expression is a literal run
    emit_literal if @literal

    @tokens
  end

  private

  # Ragel's regex-based scan of the group options introduced a lot of
  # ambiguity, so we just ask it to find the beginning of what looks
  # like an options run and handle the rest in here.
  def self.scan_options(p, data, ts, te)
    text = text(data, ts, te).first

    options_char, options_length = true, 0

    # Copy while we have option characters, the maximum is 7, for (?mix-mix,
    # even though it doesn't make sense it is possible.
    while options_char and options_length < 7
      if data[te + options_length]
        c = data[te + options_length].chr

        if c =~ /[-mix]/
          text << c ; p += 1 ; options_length += 1
        else
          options_char = false
        end
      else
        raise PrematureEndError.new("expression options `#{text}'")
      end
    end

    if data[te + options_length]
      c = data[te + options_length].chr

      if c == ':'
        # Include the ':' in the options text
        text << c ; p += 1 ; options_length += 1
        emit(:group, :options, text, ts, te + options_length)

      elsif c == ')'
        # Don't include the closing ')', let group_close handle it.
        emit(:group, :options, text, ts, te + options_length)

      else
        # Plain Regexp reports this as 'undefined group option'
        raise ScannerError.new(
          "Unexpected `#{c}' in options sequence, ':' or ')' expected")
      end
    else
      raise PrematureEndError.new("expression options `#{text}'")
    end

    p # return the new value of the data pointer
  end

  # Copy from ts to te from data as text
  def self.copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def self.text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    @literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  # Emits an array with the details of the scanned pattern
  def self.emit(type, token, text, ts, te)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if @literal

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def self.validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    else
      error = ValidationError.new('expression')
    end

    # TODO: configuration option to treat scanner level validation
    # errors as warnings or ignore them
    if false # @@config.validation_warn
      $stderr.puts error.to_s # unless @@config.validation_ignore
    else
      raise error # unless @@config.validation_ignore
    end
  end

  # Used for references with an empty name or number
  def self.empty_backref_error(type, what)
    validation_error(:backref, what, 'ref ID is empty')
  end

  # Used for named expressions with an empty name
  def self.empty_name_error(type, what)
    validation_error(type, what, 'name is empty')
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  hex_or_not            = (xdigit|[^0-9a-fA-F}]); # note closing curly at end

  wide_hex_seq_invalid  = 'x' . '{' . hex_or_not{1,9};
  wide_hex_seq_empty    = 'x' . '{' . (space+)? . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-');

  meta_sequence         = 'M-' . (backslash . control_sequence)?;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . [\-mix];

  group_ref             = [gk];
  group_name            = (alnum . (alnum+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");

  group_name_ref        = group_ref . (('<' . group_name . group_level? '>') |
                                       ("'" . group_name . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise InvalidSequenceError.new('sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { group_depth += 1; in_group = true }
  action group_closed { group_depth -= 1; in_group = group_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :close, *text(data, ts, te))

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-]' { # special case, emits two tokens
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :member, copy(data, ts..te-2), ts, te)
      emit(set_type, :close,  copy(data, ts+1..te-1), ts, te)

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '^' {
      text = text(data, ts, te).first
      if @tokens.last[1] == :open
        emit(set_type, :negate, text, ts, te)
      else
        emit(set_type, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum {
      emit(set_type, :range, *text(data, ts, te))
    };

    '&&' {
      emit(set_type, :intersection, *text(data, ts, te))
    };

    '\\' {
      fcall set_escape_sequence;
    };

    '[' >(open_bracket, 1) {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @eof(premature_end_error) {
      text = text(data, ts, te).first

      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = "non#{class_name[1..-1]}"
      end

      token_sym = "class_#{class_name}".to_sym
      emit(set_type, token_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :equivalent, *text(data, ts, te))
    };

    # exclude the closing bracket as a cleaner workaround for dealing with the
    # ambiguity caused upon exit from the unicode properties machine
    meta_char -- ']' {
      emit(set_type, :member, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      emit(set_type, :member, *text(data, ts, te))
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' > (escaped_set_alpha, 2) {
      emit(set_type, :backspace, *text(data, ts, te, 1))
      fret;
    };

    char_type > (escaped_set_alpha, 4) {
      case text = text(data, ts, te, 1).first
      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      emit(set_type, :range_hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence {
      emit(set_type, :member_hex, *text(data, ts, te, 1))
      fret;
    };

    meta_char | [\\\]\-\,] {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };

    property_char > (escaped_set_alpha, 3) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; emit(:escape, :space,          text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex_wide, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    (wide_hex_seq_invalid | wide_hex_seq_empty) {
      raise InvalidSequenceError.new("wide hex sequence")
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("control sequence")
        end
      else
        raise PrematureEndError.new("control sequence")
      end
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("meta sequence")
        end
      else
        raise PrematureEndError.new("meta sequence")
      end
      fret;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext main;
      fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      emit(:meta, :alternation, *text(data, ts, te))
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in anchor at #{text} (char #{ts})")
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = text(data, ts, te).first
      when '\\d'; emit(:type, :digit,      text, ts, te)
      when '\\D'; emit(:type, :nondigit,   text, ts, te)
      when '\\h'; emit(:type, :hex,        text, ts, te)
      when '\\H'; emit(:type, :nonhex,     text, ts, te)
      when '\\s'; emit(:type, :space,      text, ts, te)
      when '\\S'; emit(:type, :nonspace,   text, ts, te)
      when '\\w'; emit(:type, :word,       text, ts, te)
      when '\\W'; emit(:type, :nonword,    text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in type at #{text} (char #{ts})")
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      p = scan_options(p, data, ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)

      when /^\(\?<(\w*)>/
        empty_name_error(:group, 'named group (ab)') if $1.empty?

        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'(\w*)'/
        empty_name_error(:group, 'named group (sq)') if $1.empty?

        emit(:group, :named_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown subexpression group format '#{text}'")
      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      emit(:group, :close, *text(data, ts, te))
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])<>/ # angle brackets
        empty_backref_error("ref/call (ab)")

      when /^\\([gk])''/ # single quotes
        empty_backref_error("ref/call (sq)")

      when /^\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab,  text, ts, te)
        else
          emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /^\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq,  text, ts, te)
        else
          emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /^\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /^\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_rel_call_sq,  text, ts, te)
        end

      when /^\\k<[^\d-](\w+)?[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_nest_ref_ab,  text, ts, te)

      when /^\\k'[^\d-](\w+)?[+\-]\d+'/ # single-quotes
        emit(:backref, :name_nest_ref_sq,  text, ts, te)

      when /^\\([gk])<\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_nest_ref_ab,  text, ts, te)

      when /^\\([gk])'\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_nest_ref_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown backreference format '#{text}'")
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from scanner.rl

module Regexp::Scanner
  %% write data;

  # General scanner error (catch all)
  class ScannerError < StandardError
    def initialize(what)
      super what
    end
  end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group, group_depth = false, 0
    in_set,   set_depth, set_type   = false, 0, :set

    %% write init;
    %% write exec;

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{in_group}:#{group_depth}]") if in_group
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{in_set}:#{set_depth}]") if in_set

    # when the entire expression is a literal run
    emit_literal if @literal

    @tokens
  end

  private

  # Ragel's regex-based scan of the group options introduced a lot of
  # ambiguity, so we just ask it to find the beginning of what looks
  # like an options run and handle the rest in here.
  def self.scan_options(p, data, ts, te)
    text = text(data, ts, te).first

    options_char, options_length = true, 0

    # Copy while we have option characters, the maximum is 7, for (?mix-mix,
    # even though it doesn't make sense it is possible.
    while options_char and options_length < 7
      if data[te + options_length]
        c = data[te + options_length].chr

        if c =~ /[-mix]/
          text << c ; p += 1 ; options_length += 1
        else
          options_char = false
        end
      else
        raise PrematureEndError.new("expression options `#{text}'")
      end
    end

    if data[te + options_length]
      c = data[te + options_length].chr

      if c == ':'
        # Include the ':' in the options text
        text << c ; p += 1 ; options_length += 1
        emit(:group, :options, text, ts, te + options_length)

      elsif c == ')'
        # Don't include the closing ')', let group_close handle it.
        emit(:group, :options, text, ts, te + options_length)

      else
        # Plain Regexp reports this as 'undefined group option'
        raise ScannerError.new(
          "Unexpected `#{c}' in options sequence, ':' or ')' expected")
      end
    else
      raise PrematureEndError.new("expression options `#{text}'")
    end

    p # return the new value of the data pointer
  end

  # Copy from ts to te from data as text
  def self.copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def self.text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    @literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  # Emits an array with the details of the scanned pattern
  def self.emit(type, token, text, ts, te)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if @literal

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def self.validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    else
      error = ValidationError.new('expression')
    end

    # TODO: configuration option to treat scanner level validation
    # errors as warnings or ignore them
    if false # @@config.validation_warn
      $stderr.puts error.to_s # unless @@config.validation_ignore
    else
      raise error # unless @@config.validation_ignore
    end
  end

  # Used for references with an empty name or number
  def self.empty_backref_error(type, what)
    validation_error(:backref, what, 'ref ID is empty')
  end

  # Used for named expressions with an empty name
  def self.empty_name_error(type, what)
    validation_error(type, what, 'name is empty')
  end

end # module Regexp::Scanner


SSA: Don't call the parser actions for tags no renderer supports ATM.

git-svn-id: d8f21eb7283bfb39ebab2f8f9b4a259b233f9348@1196 621663c8-3916-0410-8f58-edc14a8543d5

/*
 * SubParsing.m.rl
 * Created by Alexander Strange on 7/25/07.
 *
 * This file is part of Perian.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize
{
	[delegate releaseSpanExtra:ex];
	[super finalize];
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"Span at %d: %@", offset, [delegate describeSpanEx:ex]];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(NSString *ssastr, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	unsigned len = [ssastr length];
	NSData *ssaData;
	const unichar *ssa = STUnicodeForString(ssastr, &ssaData);
	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

#define send() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)strbegin length:p-strbegin freeWhenDone:NO] autorelease]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl :> (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend %/csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl :> (format %{eventformat=str;})? <: (event*);

		main := bom? header styles lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	[ssaData release];

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = *(id*)a, *divB = *(id*)b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

#undef send
#define send()  [[[NSString alloc] initWithCharactersNoCopy:(unichar*)outputbegin length:p-outputbegin freeWhenDone:NO] autorelease]
#define psend() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)parambegin length:p-parambegin freeWhenDone:NO] autorelease]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			size_t linelen = [inputText length];
			NSData *linebufData;
			const unichar *linebuf = STUnicodeForString(inputText, &linebufData);
			const unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_tag_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; float floatnum = 0;
			NSString *strval=NULL;
			float curX, curY;
			int intnum = 0;
			BOOL reachedEnd = NO, setWrapStyle = NO, setPosition = NO, setAlignForDiv = NO, dropThisSpan = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}
				action drawingmode {tag(p, floatnum); dropThisSpan = floatnum > 0;}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action nullstring {strval = @"";}
				action setpos {curX=curY=0; sscanf([psend() UTF8String], "(%f,%f", &curX, &curY);}

				action ssaalign {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
					}
				}

				action align {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(intnum, &div->alignH, &div->alignV);
					}
				}

				action wrapstyle {
					if (!setWrapStyle) {
						setWrapStyle = YES;

						div->wrapStyle = intnum;

					}
				}

				action position {
					if (!setPosition) {
						setPosition = YES;

						div->posX = curX;
						div->posY = curY;
						div->positioned = YES;
					}
				}

				action origin {
					div->shouldResetPens = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatn = ("-"? ([0-9]+ ("." [0-9]*)?) | ([0-9]* "." [0-9]+));
				floatnum = floatn >paramset %setfloatnum;
				string = (([^\\}]+) >paramset %setstringval | "" %nullstring );
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				pos = ("(" floatn "," floatn ")") >paramset %setpos;
				move = ("(" (floatn ","){3,5} floatn ")") >paramset %setpos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" floatnum
							|"blur" floatnum
							|"fax" floatnum
							|"fay" floatnum
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum
							|"fry" floatnum
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" pos %position
							|"move" move %position
							|"t" parens
							|"org" parens %origin
							|("fad" "e"? parens)
							|"i"? "clip" parens
							|"p" floatnum %drawingmode
							|"pbo" floatnum
							|"xbord" floatnum
							|"ybord" floatnum
							|"xshad" floatnum
							|"yshad" floatnum
					   );

				tag = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					p--;
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
						}

						[div->text appendFormat:@"%C",o];
					}

					chars_deleted++;

					p++;
					outputbegin = p+1;
				}

				action enter_tag {
					if (dropThisSpan) chars_deleted += p - outputbegin;
					else if (p > outputbegin) [div->text appendString:send()];
					if (p == pe) reachedEnd = YES;

					if (p != pb) {
						[div->spans addObject:current_span];

						if (!reachedEnd) current_span = [current_span cloneWithDelegate:delegate];
					}

					last_tag_start = p;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_tag_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					p--;
				}

				special = ("\\" :> any) @backslash_handler | tag >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char+;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reachedEnd) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[linebufData release];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}

/*
 * SubParsing.m.rl
 * Created by Alexander Strange on 7/25/07.
 *
 * This file is part of Perian.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"
#import "Codecprintf.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}

-(void)finalize
{
	[delegate releaseSpanExtra:ex];
	[super finalize];
}

-(NSString*)description
{
	return [NSString stringWithFormat:@"Span at %d: %@", offset, [delegate describeSpanEx:ex]];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div \"%@\" with %d spans:", text, sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = 0;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		positioned = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

	div->positioned = positioned;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(NSString *ssastr, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	unsigned len = [ssastr length];
	NSData *ssaData;
	const unichar *ssa = STUnicodeForString(ssastr, &ssaData);
	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

#define send() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)strbegin length:p-strbegin freeWhenDone:NO] autorelease]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = ("[" [Vv] "4 Styles]") %ssaformat | ("[" [Vv] "4+ Styles]") %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl :> (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend %/csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl :> (format %{eventformat=str;})? <: (event*);

		main := bom? header styles lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	[ssaData release];

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static int compare_layer(const void *a, const void *b)
{
	const SubRenderDiv *divA = *(id*)a, *divB = *(id*)b;

	if (divA->layer < divB->layer) return -1;
	else if (divA->layer > divB->layer) return 1;
	return 0;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

#undef send
#define send()  [[[NSString alloc] initWithCharactersNoCopy:(unichar*)outputbegin length:p-outputbegin freeWhenDone:NO] autorelease]
#define psend() [[[NSString alloc] initWithCharactersNoCopy:(unichar*)parambegin length:p-parambegin freeWhenDone:NO] autorelease]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			size_t linelen = [inputText length];
			NSData *linebufData;
			const unichar *linebuf = STUnicodeForString(inputText, &linebufData);
			const unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_tag_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; float floatnum = 0;
			NSString *strval=NULL;
			float curX, curY;
			int intnum = 0;
			BOOL reachedEnd = NO, setWrapStyle = NO, setPosition = NO, setAlignForDiv = NO, dropThisSpan = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}
				action drawingmode {tag(p, floatnum); dropThisSpan = floatnum > 0;}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action nullstring {strval = @"";}
				action setpos {curX=curY=0; sscanf([psend() UTF8String], "(%f,%f", &curX, &curY);}

				action ssaalign {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
					}
				}

				action align {
					if (!setAlignForDiv) {
						setAlignForDiv = YES;

						ParseASSAlignment(intnum, &div->alignH, &div->alignV);
					}
				}

				action wrapstyle {
					if (!setWrapStyle) {
						setWrapStyle = YES;

						div->wrapStyle = intnum;

					}
				}

				action position {
					if (!setPosition) {
						setPosition = YES;

						div->posX = curX;
						div->posY = curY;
						div->positioned = YES;
					}
				}

				action origin {
					div->shouldResetPens = YES;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatn = ("-"? ([0-9]+ ("." [0-9]*)?) | ([0-9]* "." [0-9]+));
				floatnum = floatn >paramset %setfloatnum;
				string = (([^\\}]+) >paramset %setstringval | "" %nullstring );
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				pos = ("(" floatn "," floatn ")") >paramset %setpos;
				move = ("(" (floatn ","){3,5} floatn ")") >paramset %setpos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" floatnum %bluredge
							|"blur" floatnum
							|"fax" floatnum
							|"fay" floatnum
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" pos %position
							|"move" move %position
							|"t" parens
							|"org" parens %origin
							|("fad" "e"? parens)
							|"i"? "clip" parens
							|"p" floatnum %drawingmode
							|"pbo" floatnum
							|"xbord" floatnum
							|"ybord" floatnum
							|"xshad" floatnum
							|"yshad" floatnum
					   );

				tag = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					p--;
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
						}

						[div->text appendFormat:@"%C",o];
					}

					chars_deleted++;

					p++;
					outputbegin = p+1;
				}

				action enter_tag {
					if (dropThisSpan) chars_deleted += p - outputbegin;
					else if (p > outputbegin) [div->text appendString:send()];
					if (p == pe) reachedEnd = YES;

					if (p != pb) {
						[div->spans addObject:current_span];

						if (!reachedEnd) current_span = [current_span cloneWithDelegate:delegate];
					}

					last_tag_start = p;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_tag_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					p--;
				}

				special = ("\\" :> any) @backslash_handler | tag >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char+;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reachedEnd) Codecprintf(NULL, "parse error: %s\n", [inputText UTF8String]);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[linebufData release];
			[divs addObject:div];
		}

	}

	STSortMutableArrayStably(divs, compare_layer);
	return divs;
}