Fixed code confusing GCC overflow checks in unpack.

// vim: filetype=ragel

%%{

  machine unpack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    count = 0;
    rest = true;
  }

  action zero_count {
    count = 0;
  }

  action platform {
    platform = true;
  }

  action byte_width {
    width = 1;
  }

  action short_width {
    width = 2;
  }

  action int_width {
    width = 4;
  }

  action long_width {
    width = 8;
  }

  action platform_width {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      width = 4;
#else
      width = 8;
#endif
    } else {
      width = 4;
    }
  }

  action set_stop {
    if(!rest) {
      stop = index + width * count;
    }

    if(rest || stop > bytes_size) {
      stop = index + ((bytes_size - index) / width) * width;
    }
  }

  action extra {
    for(; count > 0; count--) {
      array->append(state, Qnil);
    }
  }

  # Integers

  action C {
    unpack_integer(ubyte);
  }

  action c {
    unpack_integer(sbyte);
  }

  action S {
    unpack_integer(u2bytes);
  }

  action s {
    unpack_integer(s2bytes);
  }

  action I {
    unpack_integer(u4bytes);
  }

  action i {
    unpack_integer(s4bytes);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      unpack_integer(u4bytes);
#else
      unpack_integer(u8bytes);
#endif
    } else {
      unpack_integer(u4bytes);
    }
  }

  action l {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      unpack_integer(s4bytes);
#else
      unpack_integer(s8bytes);
#endif
    } else {
      unpack_integer(s4bytes);
    }
  }

  action N {
    unpack_integer(u4bytes_be);
  }

  action n {
    unpack_integer(u2bytes_be);
  }

  action V {
    unpack_integer(u4bytes_le);
  }

  action v {
    unpack_integer(u2bytes_le);
  }

  action Q {
    unpack_integer(u8bytes);
  }

  action q {
    unpack_integer(s8bytes);
  }

  # Floats

  action D {
    unpack_double;
  }

  action E {
    unpack_double_le;
  }

  action e {
    unpack_float_le;
  }

  action F {
    unpack_float;
  }

  action G {
    unpack_double_be;
  }

  action g {
    unpack_float_be;
  }

  # Moves

  action X {
    if(rest) count = bytes_size - index;
    index -= count;
  }

  action x {
    if(rest) {
      index = bytes_size;
    } else {
      index += count;
    }
  }

  action at {
    if(!rest) {
      index = count;
    }
  }

  action check_bounds {
    if(index < 0 || index > bytes_size) {
      std::ostringstream msg;
      msg << *p << " outside of string";
      Exception::argument_error(state, msg.str().c_str());
    }
  }

  # String / Encoding helpers

  action bytes {
    bytes = (const char*)self->byte_address() + index;
  }

  action bytes_end {
    bytes_end = (const char*)self->byte_address() + bytes_size;
  }

  action string_width {
    width = 1;
  }

  action bit_width {
    width = 8;
  }

  action hex_width {
    width = 2;
  }

  action remainder {
    remainder = bytes_size - index;
  }

  action rest_count {
    if(rest) {
      count = remainder;
    } else if(count > remainder) {
      count = remainder;
    }
  }

  action string_size {
    if(rest || count > remainder * width) {
      count = remainder * width;
    }
  }

  # Strings

  action A {
    String* string;

    if(count > 0) {
      native_int i;
      for(i = count; i > 0; i--) {
        uint8_t c = bytes[i-1];
        if(c != ' ' && c != '\0')
          break;
      }
      string = String::create(state, bytes, i);
    } else {
      string = String::create(state, "");
    }

    array->append(state, string);
    unpack::increment(index, count, bytes_size);
  }

  action a {
    array->append(state, String::create(state, bytes, count));

    unpack::increment(index, count, bytes_size);
  }

  action Z {
    native_int c;
    for(c = 0; c < count; c++) {
      if(bytes[c] == '\0') break;
    }
    array->append(state, String::create(state, bytes, c));

    if(rest) {
      unpack::increment(index, c < count ? c + 1 : count, bytes_size);
    } else {
      unpack::increment(index, count, bytes_size);
    }
  }

  # Encodings

  action index_increment {
    unpack::increment(index,
                      bytes - ((const char*)self->byte_address() + index),
                      bytes_size);
  }

  action B {
    array->append(state, unpack::bit_high(state, bytes, count));
  }

  action b {
    array->append(state, unpack::bit_low(state, bytes, count));
  }

  action H {
    array->append(state, unpack::hex_high(state, bytes, count));
  }

  action h {
    array->append(state, unpack::hex_low(state, bytes, count));
  }

  action M {
    array->append(state, unpack::quotable_printable(state, bytes, bytes_end, remainder));
  }

  action m {
    array->append(state, unpack::base64_decode(state, bytes, bytes_end, remainder));
  }

  action U {
    unpack::utf8_decode(state, array, bytes, bytes_end, count, index);
  }

  action u {
    array->append(state, unpack::uu_decode(state, bytes, bytes_end, remainder));
  }

  action w {
    unpack::ber_decode(state, array, bytes, bytes_end, count, index);
    index = bytes - (const char*)self->byte_address();
  }

  action non_native_error {
    std::ostringstream msg;
    msg << "'" << *p << "' allowed only after types sSiIlL";
    Exception::argument_error(state, msg.str().c_str());
  }

  action done {
    return array;
  }
}%%

// vim: filetype=ragel

%%{

  machine unpack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    count = 0;
    rest = true;
  }

  action zero_count {
    count = 0;
  }

  action platform {
    platform = true;
  }

  action byte_width {
    width = 1;
  }

  action short_width {
    width = 2;
  }

  action int_width {
    width = 4;
  }

  action long_width {
    width = 8;
  }

  action platform_width {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      width = 4;
#else
      width = 8;
#endif
    } else {
      width = 4;
    }
  }

  action set_stop {
    stop = rest ? bytes_size + 1 : index + width * count;
    if(stop > bytes_size) {
      stop = index + ((bytes_size - index) / width) * width;
    }
  }

  action extra {
    for(; count > 0; count--) {
      array->append(state, Qnil);
    }
  }

  # Integers

  action C {
    unpack_integer(ubyte);
  }

  action c {
    unpack_integer(sbyte);
  }

  action S {
    unpack_integer(u2bytes);
  }

  action s {
    unpack_integer(s2bytes);
  }

  action I {
    unpack_integer(u4bytes);
  }

  action i {
    unpack_integer(s4bytes);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      unpack_integer(u4bytes);
#else
      unpack_integer(u8bytes);
#endif
    } else {
      unpack_integer(u4bytes);
    }
  }

  action l {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      unpack_integer(s4bytes);
#else
      unpack_integer(s8bytes);
#endif
    } else {
      unpack_integer(s4bytes);
    }
  }

  action N {
    unpack_integer(u4bytes_be);
  }

  action n {
    unpack_integer(u2bytes_be);
  }

  action V {
    unpack_integer(u4bytes_le);
  }

  action v {
    unpack_integer(u2bytes_le);
  }

  action Q {
    unpack_integer(u8bytes);
  }

  action q {
    unpack_integer(s8bytes);
  }

  # Floats

  action D {
    unpack_double;
  }

  action E {
    unpack_double_le;
  }

  action e {
    unpack_float_le;
  }

  action F {
    unpack_float;
  }

  action G {
    unpack_double_be;
  }

  action g {
    unpack_float_be;
  }

  # Moves

  action X {
    if(rest) count = bytes_size - index;
    index -= count;
  }

  action x {
    if(rest) {
      index = bytes_size;
    } else {
      index += count;
    }
  }

  action at {
    if(!rest) {
      index = count;
    }
  }

  action check_bounds {
    if(index < 0 || index > bytes_size) {
      std::ostringstream msg;
      msg << *p << " outside of string";
      Exception::argument_error(state, msg.str().c_str());
    }
  }

  # String / Encoding helpers

  action bytes {
    bytes = (const char*)self->byte_address() + index;
  }

  action bytes_end {
    bytes_end = (const char*)self->byte_address() + bytes_size;
  }

  action string_width {
    width = 1;
  }

  action bit_width {
    width = 8;
  }

  action hex_width {
    width = 2;
  }

  action remainder {
    remainder = bytes_size - index;
  }

  action rest_count {
    if(rest) {
      count = remainder;
    } else if(count > remainder) {
      count = remainder;
    }
  }

  action string_size {
    if(rest || count > remainder * width) {
      count = remainder * width;
    }
  }

  # Strings

  action A {
    String* string;

    if(count > 0) {
      native_int i;
      for(i = count; i > 0; i--) {
        uint8_t c = bytes[i-1];
        if(c != ' ' && c != '\0')
          break;
      }
      string = String::create(state, bytes, i);
    } else {
      string = String::create(state, "");
    }

    array->append(state, string);
    unpack::increment(index, count, bytes_size);
  }

  action a {
    array->append(state, String::create(state, bytes, count));

    unpack::increment(index, count, bytes_size);
  }

  action Z {
    native_int c;
    for(c = 0; c < count; c++) {
      if(bytes[c] == '\0') break;
    }
    array->append(state, String::create(state, bytes, c));

    if(rest) {
      unpack::increment(index, c < count ? c + 1 : count, bytes_size);
    } else {
      unpack::increment(index, count, bytes_size);
    }
  }

  # Encodings

  action index_increment {
    unpack::increment(index,
                      bytes - ((const char*)self->byte_address() + index),
                      bytes_size);
  }

  action B {
    array->append(state, unpack::bit_high(state, bytes, count));
  }

  action b {
    array->append(state, unpack::bit_low(state, bytes, count));
  }

  action H {
    array->append(state, unpack::hex_high(state, bytes, count));
  }

  action h {
    array->append(state, unpack::hex_low(state, bytes, count));
  }

  action M {
    array->append(state, unpack::quotable_printable(state, bytes, bytes_end, remainder));
  }

  action m {
    array->append(state, unpack::base64_decode(state, bytes, bytes_end, remainder));
  }

  action U {
    unpack::utf8_decode(state, array, bytes, bytes_end, count, index);
  }

  action u {
    array->append(state, unpack::uu_decode(state, bytes, bytes_end, remainder));
  }

  action w {
    unpack::ber_decode(state, array, bytes, bytes_end, count, index);
    index = bytes - (const char*)self->byte_address();
  }

  action non_native_error {
    std::ostringstream msg;
    msg << "'" << *p << "' allowed only after types sSiIlL";
    Exception::argument_error(state, msg.str().c_str());
  }

  action done {
    return array;
  }
}%%


Cleaned up the Ragel bits of the lexer.

Cleaned up the Ragel bits of the lexer.

This removes some of the complexity that existed before (e.g. too many state
machines) and fixes a bunch of problems with nested data.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    def reset
      @line     = 1
      @column   = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    def html?
      return !!html
    end

    private

    def advance_line(amount = 1)
      @line  += amount
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length) if value

      @tokens << token
    end

    def emit_text_buffer
      return if @text_buffer.empty?

      add_token(:T_TEXT, @text_buffer)

      lines = @text_buffer.count("\n")

      advance_line(lines) if lines > 0

      @text_buffer = ''
    end

    def buffer_text_until_eof(eof)
      @text_buffer << text

      emit_text_buffer if @te == eof
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # Strings
      #
      # Strings in HTML can either be single or double quoted. If a string
      # starts with one of these quotes it must be closed with the same type of
      # quote.
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action start_string_dquote {
        fcall string_dquote;
      }

      action start_string_squote {
        fcall string_squote;
      }

      # Machine for processing double quoted strings.
      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # Machine for processing single quoted strings.
      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      action start_doctype {
        emit_text_buffer
        t(:T_DOCTYPE_START)
        fcall doctype;
      }

      # Machine for processing doctypes. Doctype values such as the public and
      # system IDs are treated as T_STRING tokens.
      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      action start_cdata {
        emit_text_buffer
        t(:T_CDATA_START)
        fcall cdata;
      }

      # Machine that for processing the contents of CDATA tags. Everything
      # inside a CDATA tag is treated as plain text.
      cdata := |*
        cdata_end => {
          emit_text_buffer
          t(:T_CDATA_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      action start_comment {
        emit_text_buffer
        t(:T_COMMENT_START)
        fcall comment;
      }

      # Machine used for processing the contents of a comment. Everything
      # inside a comment is treated as plain text (similar to CDATA tags).
      comment := |*
        comment_end => {
          emit_text_buffer
          t(:T_COMMENT_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action start_element {
        emit_text_buffer
        add_token(:T_ELEM_OPEN, nil)
        advance_column

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)

          # Advance the column for the colon (:) that separates the namespace
          # and element name.
          advance_column
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element_head;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      # Machine used for processing the characters inside a element head. An
      # element head is everything between `<NAME` (where NAME is the element
      # name) and `>`.
      #
      # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
      #
      element_head := |*
        (whitespace | '=') => { advance_column };

        # Attribute names.
        element_name => { t(:T_ATTR) };

        # Attribute values.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # The closing character of the open tag.
        ('>' | '/') => {
          fhold;
          fret;
        };
      *|;

      main := |*
        element_start => start_element;
        doctype_start => start_doctype;
        cdata_start   => start_cdata;
        comment_start => start_comment;
        element_start => start_element;

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_CLOSE, nil)
            @elements.pop
          end

          advance_column
        };

        # Regular closing tags.
        '</' element_name '>' => {
          emit_text_buffer
          add_token(:T_ELEM_CLOSE, nil)

          advance_column(@te - @ts)

          @elements.pop
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
          advance_column
          add_token(:T_ELEM_CLOSE, nil)

          @elements.pop
        };

        # Note that this rule should be declared at the very bottom as it will
        # otherwise take precedence over the other rules.
        any => { buffer_text_until_eof(eof) };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    def reset
      @line     = 1
      @column   = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    def html?
      return !!html
    end

    private

    def advance_line(amount = 1)
      @line  += amount
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length) if value

      @tokens << token
    end

    def emit_text_buffer
      return if @text_buffer.empty?

      add_token(:T_TEXT, @text_buffer)

      lines = @text_buffer.count("\n")

      advance_line(lines) if lines > 0

      @text_buffer = ''
    end

    def buffer_text_until_eof(eof)
      @text_buffer << text

      emit_text_buffer if @te == eof
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        fcall string_dquote;
      }

      action string_squote {
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer
          t(:T_CDATA_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      comment := |*
        comment_end => {
          emit_text_buffer
          t(:T_COMMENT_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action open_element {
        emit_text_buffer
        add_token(:T_ELEM_OPEN, nil)
        advance_column

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)

          # Advance the column for the colon (:) that separates the namespace
          # and element name.
          advance_column
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      element_text := |*
        ^'<' => {
          buffer_text_until_eof(eof)
        };

        '<' => {
          emit_text_buffer
          fhold;
          fret;
        };
      *|;

      element_closing_tag := |*
        whitespace => { advance_column };

        element_name => {
          emit_text_buffer
          add_token(:T_ELEM_CLOSE, nil)

          # Advance the column for the </
          advance_column(2)

          # Advance the column for the closing name.
          advance_column(text.length)

          fret;
        };

        '>' => { fret; };
      *|;

      element := |*
        whitespace => { advance_column };

        element_start => open_element;

        # Consume the text inside the element.
        '>' => {
          # If HTML lexing is enabled and we're in a void element we'll bail
          # out right away.
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_CLOSE, nil)
            @elements.pop
          end

          advance_column

          fcall element_text;
        };

        # Attributes and their values.
        element_name %{ t(:T_ATTR, @ts, p) }

        # The value of the attribute. Attribute values are not required. e.g.
        # in <p data-foo></p> data-foo would be a boolean attribute.
        (
          '=' >{ advance_column }

          # The value of the attribute, wrapped in either single or double
          # quotes.
          (dquote @string_dquote | squote @string_squote)
        )*;

        # Non self-closing elements.
        '</' => {
          fcall element_closing_tag;

          @elements.pop

          fret;
        };

        # self-closing / void elements.
        '/>' => {
          advance_column
          add_token(:T_ELEM_CLOSE, nil)

          @elements.pop

          fret;
        };
      *|;

      main := |*
        doctype_start => {
          emit_text_buffer
          t(:T_DOCTYPE_START)
          fcall doctype;
        };

        cdata_start => {
          emit_text_buffer
          t(:T_CDATA_START)
          fcall cdata;
        };

        comment_start => {
          emit_text_buffer
          t(:T_COMMENT_START)
          fcall comment;
        };

        element_start => open_element;

        any => {
          buffer_text_until_eof(eof)
        };
      *|;
    }%%
  end # Lexer
end # Oga


SSA: \r shouldn't reset positioning

SSA: \r shouldn't reset positioning

/*
 *  SSARenderCodec.m
 *  Copyright (c) 2007 Perian Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation;
 *  version 2.1 of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */

#import "SSATagParsing.h"
#import "Categories.h"

@implementation SSAStyleSpan
-(void)dealloc
{
	ATSUDisposeStyle(astyle);
	[super dealloc];
}
@end

@implementation SSARenderEntity
-(void)dealloc
{
	int i;
	for (i=0; i < style_count; i++) [styles[i] release];
	if (disposelayout) ATSUDisposeTextLayout(layout);
	free(text);
	free(styles);
	[nstext release];
	[super dealloc];
}

-(void)increasestyles
{
	style_count++;
	styles = realloc(styles, sizeof(SSAStyleSpan*[style_count]));
}
@end

%% machine SSAtag;
%% write data;

void SetATSUStyleFlag(ATSUStyle style, ATSUAttributeTag t, Boolean v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {sizeof(v)};
	ATSUAttributeValuePtr vals[] = {&v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSUStyleOther(ATSUStyle style, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSULayoutOther(ATSUTextLayout l, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetLayoutControls(l,1,tags,sizes,vals);
}

static ATSURGBAlphaColor ParseColorTag(unsigned long c, float a)
{
	unsigned char r,g,b;
	r = c & 0xff;
	g = (c >> 8) & 0xff;
	b = (c >> 16) & 0xff;

	return (ATSURGBAlphaColor){r/255.,g/255.,b/255.,a};
}

//not worth it
/*
static void PruneIdenticalStyleSpans(SSARenderEntity *re)
{
	SSAStyleSpan *styles_new[re->style_count];

	if (re->multipart_drawing || re->style_count <= 1) return;

	size_t style_count_new = 1, remaining = re->style_count-1;
	int i=1;

	styles_new[0] = re->styles[0];

	while (remaining) {
		ATSUStyleComparison asc;
		ATSUCompareStyles(styles_new[style_count_new-1]->astyle,re->styles[i]->astyle,&asc);

		if (asc == kATSUStyleEquals) {
            styles_new[style_count_new-1]->range.length += re->styles[i]->range.length;
			[re->styles[i] release];
            re->styles[i] = nil;
		} else {
			styles_new[style_count_new++] = re->styles[i];
		}

		i++;
		remaining--;
	}

	if (style_count_new != re->style_count) {
		re->styles = realloc(re->styles, sizeof(SSAStyleSpan*) * style_count_new);
		memcpy(re->styles, styles_new, sizeof(SSAStyleSpan*) * style_count_new);

		re->style_count = style_count_new;
	}
}
*/

static void UpdateAlignment(int inum, int cur_posx, int *valign, int *halign, ATSUTextLayout cur_layout)
{
	int cur_halign, cur_valign;

	switch (inum)
	{case 1: case 4: case 7: cur_halign = S_LeftAlign; break;
	case 2: case 5: case 8: default: cur_halign = S_CenterAlign; break;
	case 3: case 6: case 9: cur_halign = S_RightAlign;}

	switch (inum)
	{case 1: case 2: case 3: default: cur_valign = S_BottomAlign; break;
	case 4: case 5: case 6: cur_valign = S_MiddleAlign; break;
	case 7: case 8: case 9: cur_valign = S_TopAlign;}

	*halign = cur_halign;
	*valign = cur_valign;
	Fract alignment;

	if (cur_posx != -1) {
		switch(cur_halign) {
			case S_LeftAlign:
				alignment = FloatToFract(0.);
				break;
			case S_CenterAlign: default:
				alignment = kATSUCenterAlignment;
				break;
			case S_RightAlign:
				alignment = FloatToFract(1.);
		}
		SetATSULayoutOther(cur_layout, kATSULineFlushFactorTag, sizeof(Fract), &alignment);
	}
}

NSArray *ParseSubPacket(NSString *str, SSADocument *ssa, Boolean plaintext)
{
	NSArray *linea;
	int i=0, j, pcount = 1; unsigned len;
	NSMutableArray *rentities = [NSMutableArray array];

	if (plaintext) {
		linea = [NSArray arrayWithObject:str];
	} else {
		linea =  [str componentsSeparatedByString:@"\n"];
		pcount = [linea count];
	}

	for (; i < pcount; i ++) {
		SSARenderEntity *re = [[[SSARenderEntity alloc] init] autorelease];
		BOOL ldirty = NO;

		if (plaintext) {
			re->style = ssa->defaultstyle;
			re->layout = re->style->layout;
			re->layer = 0;
			re->marginl = re->style->marginl;
			re->marginr = re->style->marginl;
			re->marginv = re->style->marginl;
			re->usablewidth = re->style->usablewidth;
			re->halign = 1;
			re->valign = 0;
			re->nstext = [linea objectAtIndex:i];
			re->styles = NULL;
			re->disposelayout = NO;
		} else {
			NSArray *ar = [[linea objectAtIndex:(ssa->collisiontype == NormalCollisions) ? i : (pcount - i - 1)]
						   componentsSeparatedByString:@"," count:9];

			if ([ar count] < 9) continue;

			re->layer = [[ar objectAtIndex:1] intValue];
			NSString *sn = [ar objectAtIndex:2];

			re->style = ssa->defaultstyle;

			for (j=0; j < ssa->stylecount; j++) {
				if ([sn isEqualToString:ssa->styles[j].name])  {
					re->style = &ssa->styles[j];
					break;
				}
			}

			re->marginl = [[ar objectAtIndex:5] intValue];
			re->marginr = [[ar objectAtIndex:6] intValue];
			re->marginv = [[ar objectAtIndex:7] intValue];

			if (re->marginl == 0) re->marginl = re->style->marginl; else ldirty = TRUE;
			if (re->marginr == 0) re->marginr = re->style->marginr; else ldirty = TRUE;
			if (re->marginv == 0) re->marginv = re->style->marginv; else ldirty = TRUE;

			if (ldirty) {
				ATSUTextMeasurement width;
				ATSUAttributeTag tag[] = {kATSULineWidthTag};
				ByteCount		 size[] = {sizeof(ATSUTextMeasurement)};
				ATSUAttributeValuePtr val[] = {&width};

				re->usablewidth = ssa->resX - re->marginl - re->marginr;
				ATSUCreateAndCopyTextLayout(re->style->layout,&re->layout);
				width = IntToFixed(re->usablewidth);

				ATSUSetLayoutControls(re->layout,1,tag,size,val);
				re->disposelayout = YES;
			} else {
				re->usablewidth = re->style->usablewidth;
				re->layout = re->style->layout;
				re->disposelayout = NO;
			}

			re->nstext = [ar objectAtIndex:8];
		}
		len = [re->nstext length];
		re->text = malloc(sizeof(unichar[len]));

		[re->nstext getCharacters:re->text];
		re->style_count = 0;
		re->posx = re->posy = -1;
		re->halign = re->style->halign;
		re->valign = re->style->valign;
		re->multipart_drawing=NO;
		re->is_shape=NO;

#define end_re \
		cur_range = (NSRange){strbegin - pb, p - strbegin};\
		cur_range.location -= outputoffset;\
		cur_range.length -= lengthreduce;\
		if (cur_range.length) {[re increasestyles];\
		re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];\
		re->styles[re->style_count-1]->outline = cur_outline;\
		re->styles[re->style_count-1]->shadow = cur_shadow;\
		re->styles[re->style_count-1]->angle = cur_frz;\
		re->styles[re->style_count-1]->astyle = cur_style;\
		re->styles[re->style_count-1]->range = cur_range;\
		re->styles[re->style_count-1]->outlineblur = cur_be;\
		re->styles[re->style_count-1]->color = cur_color;}\
		parsetmp = [NSString stringWithCharacters:skipbegin length:p-skipbegin];\
		[output appendString:parsetmp]; \
		re->nstext = dtmp = output;\
		re->text = malloc(sizeof(unichar[[re->nstext length]]));\
		[re->nstext getCharacters:re->text];\
		[re->nstext retain];

		{
			NSRange cur_range = {0,0};
			ATSUStyle cur_style;
			ATSUTextLayout cur_layout;
			NSMutableString *output = [NSMutableString string], *dtmp;
			unichar *p = re->text, *pe = &re->text[len], *numbegin = p, *strbegin = p, *skipbegin = p, *intbegin = p, *pb = p, *posbegin=p, *strparambegin=p;
			float num, cur_outline = re->style->outline, cur_shadow = re->style->shadow, cur_scalex = re->style->scalex, cur_scaley = re->style->scaley, cur_frz=re->style->angle;
			NSString *parsetmp;
			int cs, cur_valign=re->valign, cur_halign=re->halign, cur_posx=-1, cur_posy=-1;
			ssacolors cur_color = re->style->color;
			CGAffineTransform matrix;

			unsigned long inum=0;
			unsigned outputoffset=0, lengthreduce=0;
			BOOL flag=0, newLayout=FALSE, cur_be = FALSE;
			Fixed fixv;

			ATSUCreateAndCopyStyle(re->style->atsustyle,&cur_style);

			%%{
				alphtype unsigned short;

				action bold {SetATSUStyleFlag(cur_style, kATSUQDBoldfaceTag, flag);}

				action italic {SetATSUStyleFlag(cur_style, kATSUQDItalicTag, flag);}

				action uline {SetATSUStyleFlag(cur_style, kATSUQDUnderlineTag, flag);}

				action strike {SetATSUStyleFlag(cur_style, kATSUStyleStrikeThroughTag, flag);}

				action bordersize {cur_outline = num; re->multipart_drawing = YES;}

				action shadowsize {cur_shadow = num; re->multipart_drawing = YES;}

				action bluredge {cur_be = flag;}

				action frot {cur_frz = num; re->multipart_drawing = YES;}

				action fsize {
					num *= (72./96.); // scale from Windows 96dpi
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUSizeTag, sizeof(fixv), &fixv);
				}

				action ftrack {
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUTrackingTag, sizeof(fixv), &fixv);
				}

				action fscalex {
					cur_scalex = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action fscaley {
					cur_scaley = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action strp_begin {
					strparambegin = p;
				}

				action fontname {
					ATSUFontID	font;
					font = FMGetFontFromATSFontRef(ATSFontFindFromName((CFStringRef)[NSString stringWithCharacters:strparambegin length:p-strparambegin],kATSOptionFlagsDefault));
					SetATSUStyleOther(cur_style, kATSUFontTag, sizeof(ATSUFontID), &font);
				}

				action alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(inum, cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action ssa_alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(SSA2ASSAlignment(inum), cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action pos_begin {
					posbegin=p;
				}

				action pos_end {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}
					NSArray *coo = [[NSString stringWithCharacters:posbegin length:p-posbegin] componentsSeparatedByString:@","];
					cur_posx = [[coo objectAtIndex:0] intValue];
					cur_posy = [[coo objectAtIndex:1] intValue];
				}

				action primarycolor {
					cur_color.primary = ParseColorTag(inum,cur_color.primary.alpha);
					re->multipart_drawing = YES;
				}

				action secondarycolor {
					cur_color.secondary = ParseColorTag(inum,cur_color.secondary.alpha);
					re->multipart_drawing = YES;
				}

				action outlinecolor {
					cur_color.outline = ParseColorTag(inum,cur_color.outline.alpha);
					re->multipart_drawing = YES;
				}

				action shadowcolor {
					cur_color.shadow = ParseColorTag(inum,cur_color.shadow.alpha);
					re->multipart_drawing = YES;
				}

				action primaryalpha {
					cur_color.primary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action secondaryalpha {
					cur_color.secondary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action outlinealpha {
					cur_color.outline.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action shadowalpha {
					cur_color.shadow.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action nl_handler {
					NSString *append = @"\n";
					parsetmp = [NSString stringWithCharacters:skipbegin length:(p-2)-skipbegin];
					[output appendString:parsetmp];

					skipbegin = p;

					if (*(p - 1) == 'h') append = @" ";

					[output appendString:append];

					lengthreduce++;
				}

				action enter_tag {
					unsigned taglen = p - skipbegin;
					parsetmp = [NSString stringWithCharacters:skipbegin length:taglen];
					[output appendString:parsetmp];
					skipbegin = p;

					cur_range = (NSRange){strbegin - pb, p - strbegin};
					cur_range.location -= outputoffset;
					cur_range.length -= lengthreduce;

					outputoffset += lengthreduce;
					lengthreduce = 0;

                    if (cur_range.length) {
                        [re increasestyles];
                        re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];
                        re->styles[re->style_count-1]->outline = cur_outline;
                        re->styles[re->style_count-1]->shadow = cur_shadow;
                        re->styles[re->style_count-1]->angle = cur_frz;
                        re->styles[re->style_count-1]->astyle = cur_style;
                        re->styles[re->style_count-1]->range = cur_range;
                        re->styles[re->style_count-1]->outlineblur = cur_be;
                        re->styles[re->style_count-1]->color = cur_color;
                        ATSUCreateAndCopyStyle(cur_style,&cur_style);
                    }
				}

				action exit_tag {
					// XXX exit_tag ideally should run one char later, so we just pretend it does
					p++;
					outputoffset += p - skipbegin;
					skipbegin = p;
					strbegin = p;

					if (newLayout) {
						newLayout = FALSE;
						end_re;

						if ([re->nstext length] != 0) [rentities addObject:re];

						SSARenderEntity *nre = [[[SSARenderEntity alloc] init] autorelease];
						nre->layer = re->layer;
						nre->style = re->style;
						nre->marginl = re->marginl; nre->marginr = re->marginr; nre->marginv = re->marginv; nre->usablewidth = re->usablewidth;
						nre->posx = cur_posx; nre->posy = cur_posy; nre->halign = cur_halign; nre->valign = cur_valign;
						nre->nstext = nil;
						nre->text = nil;
						nre->layout = cur_layout;
						nre->styles = malloc(0);
						nre->style_count = 0;
						nre->disposelayout = YES;
						nre->multipart_drawing = NO;
						nre->is_shape = re->is_shape;
						re = nre;
					}
					p--;
				}

				action color_end {
					NSString *hexn = [NSString stringWithCharacters:intbegin length:p-intbegin];
					inum = strtoul([hexn UTF8String], NULL, 16);
				}

				action styleset {
					re->multipart_drawing = YES;
					ssastyleline *the_style = re->style;
					NSString *searchsn = [NSString stringWithCharacters:strparambegin length:p-strparambegin];

					if ([searchsn length] > 0) {
						for (j=0; j < ssa->stylecount; j++) {
							if ([searchsn isEqualToString:ssa->styles[j].name])  {
								the_style = &ssa->styles[j];
								break;
							}
						}
					}

					ATSUCopyAttributes(the_style->atsustyle,cur_style);
					cur_color = the_style->color;
					cur_outline = the_style->outline;
					cur_shadow = the_style->shadow;
					cur_frz = the_style->angle;
					cur_scalex = cur_scaley = 1;
				}

				action skip_t_tag {
					while (p != pe && *p != ')' && *p != '}') p++;
					p++;
				}

				action draw_mode {
					re->is_shape = inum != 0;
				}

				flag = ([01] % {unichar fl = *(p-1); if (fl == '0' || fl == '1') flag = fl - '0';})? > {flag = 0;};
				num_ = "-"? digit+ ('.' digit*)?;
				num = num_ > {numbegin = p;} % {num = [[NSString stringWithCharacters:numbegin length:p-numbegin] doubleValue];};

				intnum = "-"? (digit+) > {intbegin = p;} % {inum = [[NSString stringWithCharacters:intbegin length:p-intbegin] intValue];};

				color = ("H"|"&"){,2} (xdigit+) > {intbegin = p;} % color_end "&";

				cmd_specific = (("pos(" [^)]+ > pos_begin ")" % pos_end)
								|"bord" num %bordersize
								|"b" flag %bold
								|"be" flag %bluredge
								|"i" flag %italic
								|"u" flag %uline
								|"s" flag %strike
								|"fs" num %fsize
								|"fsp" num %ftrack
								|"fscx" num %fscalex
								|"fscy" num %fscaley
								|("fr" "z"? num %frot)
								|("fn" [^\\}]* > strp_begin %fontname)
								|"shad" num %shadowsize
								|"a" intnum %ssa_alignment
								|"an" intnum %alignment
								|"c" color %primarycolor
								|"1c" color %primarycolor
								|"2c" color %secondarycolor
								|"3c" color %outlinecolor
								|"4c" color %shadowcolor
								|"1a" color %primaryalpha
								|"2a" color %secondaryalpha
								|"3a" color %outlinealpha
								|"4a" color %shadowalpha
								|("r" [^\\}]* > strp_begin %styleset)
								|("fe"|"k"|"kf"|"K"|"ko"|"q"|"fr"|"fad"|"move"|"clip"|"o"|"frx"|"fry") [^\\}]*
								|"p" num %draw_mode
								#|"t" [^)}]* # enabling this crashes ragel
								|"t(" % skip_t_tag
								);

				cmd = ("\\" :> cmd_specific)+;

				plaintext = [^}]*;

				tag = "{" (cmd | plaintext) % exit_tag "}";

				nl = "\\" [Nnh];

				special = nl % nl_handler | (tag > enter_tag);

				text = any*;
				main := (text special?)*;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (pb[len-1] == '}') skipbegin = p; // make up for how exit_tag isn't called if the } is the last char in the line

			end_re;
			if (!cur_range.length) ATSUDisposeStyle(cur_style);
            //PruneIdenticalStyleSpans(re);

			free(pb);
			if ([re->nstext length] != 0) [rentities addObject:re];
		}

	}
	return rentities;
}

/*
 *  SSARenderCodec.m
 *  Copyright (c) 2007 Perian Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation;
 *  version 2.1 of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */

#import "SSATagParsing.h"
#import "Categories.h"

@implementation SSAStyleSpan
-(void)dealloc
{
	ATSUDisposeStyle(astyle);
	[super dealloc];
}
@end

@implementation SSARenderEntity
-(void)dealloc
{
	int i;
	for (i=0; i < style_count; i++) [styles[i] release];
	if (disposelayout) ATSUDisposeTextLayout(layout);
	free(text);
	free(styles);
	[nstext release];
	[super dealloc];
}

-(void)increasestyles
{
	style_count++;
	styles = realloc(styles, sizeof(SSAStyleSpan*[style_count]));
}
@end

%% machine SSAtag;
%% write data;

void SetATSUStyleFlag(ATSUStyle style, ATSUAttributeTag t, Boolean v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {sizeof(v)};
	ATSUAttributeValuePtr vals[] = {&v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSUStyleOther(ATSUStyle style, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetAttributes(style,1,tags,sizes,vals);
}

void SetATSULayoutOther(ATSUTextLayout l, ATSUAttributeTag t, ByteCount s, const ATSUAttributeValuePtr v)
{
	ATSUAttributeTag tags[] = {t};
	ByteCount		 sizes[] = {s};
	ATSUAttributeValuePtr vals[] = {v};

	ATSUSetLayoutControls(l,1,tags,sizes,vals);
}

static ATSURGBAlphaColor ParseColorTag(unsigned long c, float a)
{
	unsigned char r,g,b;
	r = c & 0xff;
	g = (c >> 8) & 0xff;
	b = (c >> 16) & 0xff;

	return (ATSURGBAlphaColor){r/255.,g/255.,b/255.,a};
}

//not worth it
/*
static void PruneIdenticalStyleSpans(SSARenderEntity *re)
{
	SSAStyleSpan *styles_new[re->style_count];

	if (re->multipart_drawing || re->style_count <= 1) return;

	size_t style_count_new = 1, remaining = re->style_count-1;
	int i=1;

	styles_new[0] = re->styles[0];

	while (remaining) {
		ATSUStyleComparison asc;
		ATSUCompareStyles(styles_new[style_count_new-1]->astyle,re->styles[i]->astyle,&asc);

		if (asc == kATSUStyleEquals) {
            styles_new[style_count_new-1]->range.length += re->styles[i]->range.length;
			[re->styles[i] release];
            re->styles[i] = nil;
		} else {
			styles_new[style_count_new++] = re->styles[i];
		}

		i++;
		remaining--;
	}

	if (style_count_new != re->style_count) {
		re->styles = realloc(re->styles, sizeof(SSAStyleSpan*) * style_count_new);
		memcpy(re->styles, styles_new, sizeof(SSAStyleSpan*) * style_count_new);

		re->style_count = style_count_new;
	}
}
*/

static void UpdateAlignment(int inum, int cur_posx, int *valign, int *halign, ATSUTextLayout cur_layout)
{
	int cur_halign, cur_valign;

	switch (inum)
	{case 1: case 4: case 7: cur_halign = S_LeftAlign; break;
	case 2: case 5: case 8: default: cur_halign = S_CenterAlign; break;
	case 3: case 6: case 9: cur_halign = S_RightAlign;}

	switch (inum)
	{case 1: case 2: case 3: default: cur_valign = S_BottomAlign; break;
	case 4: case 5: case 6: cur_valign = S_MiddleAlign; break;
	case 7: case 8: case 9: cur_valign = S_TopAlign;}

	*halign = cur_halign;
	*valign = cur_valign;
	Fract alignment;

	if (cur_posx != -1) {
		switch(cur_halign) {
			case S_LeftAlign:
				alignment = FloatToFract(0.);
				break;
			case S_CenterAlign: default:
				alignment = kATSUCenterAlignment;
				break;
			case S_RightAlign:
				alignment = FloatToFract(1.);
		}
		SetATSULayoutOther(cur_layout, kATSULineFlushFactorTag, sizeof(Fract), &alignment);
	}
}

NSArray *ParseSubPacket(NSString *str, SSADocument *ssa, Boolean plaintext)
{
	NSArray *linea;
	int i=0, j, pcount = 1; unsigned len;
	NSMutableArray *rentities = [NSMutableArray array];

	if (plaintext) {
		linea = [NSArray arrayWithObject:str];
	} else {
		linea =  [str componentsSeparatedByString:@"\n"];
		pcount = [linea count];
	}

	for (; i < pcount; i ++) {
		SSARenderEntity *re = [[[SSARenderEntity alloc] init] autorelease];
		BOOL ldirty = NO;

		if (plaintext) {
			re->style = ssa->defaultstyle;
			re->layout = re->style->layout;
			re->layer = 0;
			re->marginl = re->style->marginl;
			re->marginr = re->style->marginl;
			re->marginv = re->style->marginl;
			re->usablewidth = re->style->usablewidth;
			re->halign = 1;
			re->valign = 0;
			re->nstext = [linea objectAtIndex:i];
			re->styles = NULL;
			re->disposelayout = NO;
		} else {
			NSArray *ar = [[linea objectAtIndex:(ssa->collisiontype == NormalCollisions) ? i : (pcount - i - 1)]
						   componentsSeparatedByString:@"," count:9];

			if ([ar count] < 9) continue;

			re->layer = [[ar objectAtIndex:1] intValue];
			NSString *sn = [ar objectAtIndex:2];

			re->style = ssa->defaultstyle;

			for (j=0; j < ssa->stylecount; j++) {
				if ([sn isEqualToString:ssa->styles[j].name])  {
					re->style = &ssa->styles[j];
					break;
				}
			}

			re->marginl = [[ar objectAtIndex:5] intValue];
			re->marginr = [[ar objectAtIndex:6] intValue];
			re->marginv = [[ar objectAtIndex:7] intValue];

			if (re->marginl == 0) re->marginl = re->style->marginl; else ldirty = TRUE;
			if (re->marginr == 0) re->marginr = re->style->marginr; else ldirty = TRUE;
			if (re->marginv == 0) re->marginv = re->style->marginv; else ldirty = TRUE;

			if (ldirty) {
				ATSUTextMeasurement width;
				ATSUAttributeTag tag[] = {kATSULineWidthTag};
				ByteCount		 size[] = {sizeof(ATSUTextMeasurement)};
				ATSUAttributeValuePtr val[] = {&width};

				re->usablewidth = ssa->resX - re->marginl - re->marginr;
				ATSUCreateAndCopyTextLayout(re->style->layout,&re->layout);
				width = IntToFixed(re->usablewidth);

				ATSUSetLayoutControls(re->layout,1,tag,size,val);
				re->disposelayout = YES;
			} else {
				re->usablewidth = re->style->usablewidth;
				re->layout = re->style->layout;
				re->disposelayout = NO;
			}

			re->nstext = [ar objectAtIndex:8];
		}
		len = [re->nstext length];
		re->text = malloc(sizeof(unichar[len]));

		[re->nstext getCharacters:re->text];
		re->style_count = 0;
		re->posx = re->posy = -1;
		re->halign = re->style->halign;
		re->valign = re->style->valign;
		re->multipart_drawing=NO;
		re->is_shape=NO;

#define end_re \
		cur_range = (NSRange){strbegin - pb, p - strbegin};\
		cur_range.location -= outputoffset;\
		cur_range.length -= lengthreduce;\
		if (cur_range.length) {[re increasestyles];\
		re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];\
		re->styles[re->style_count-1]->outline = cur_outline;\
		re->styles[re->style_count-1]->shadow = cur_shadow;\
		re->styles[re->style_count-1]->angle = cur_frz;\
		re->styles[re->style_count-1]->astyle = cur_style;\
		re->styles[re->style_count-1]->range = cur_range;\
		re->styles[re->style_count-1]->outlineblur = cur_be;\
		re->styles[re->style_count-1]->color = cur_color;}\
		parsetmp = [NSString stringWithCharacters:skipbegin length:p-skipbegin];\
		[output appendString:parsetmp]; \
		re->nstext = dtmp = output;\
		re->text = malloc(sizeof(unichar[[re->nstext length]]));\
		[re->nstext getCharacters:re->text];\
		[re->nstext retain];

		{
			NSRange cur_range = {0,0};
			ATSUStyle cur_style;
			ATSUTextLayout cur_layout;
			NSMutableString *output = [NSMutableString string], *dtmp;
			unichar *p = re->text, *pe = &re->text[len], *numbegin = p, *strbegin = p, *skipbegin = p, *intbegin = p, *pb = p, *posbegin=p, *strparambegin=p;
			float num, cur_outline = re->style->outline, cur_shadow = re->style->shadow, cur_scalex = re->style->scalex, cur_scaley = re->style->scaley, cur_frz=re->style->angle;
			NSString *parsetmp;
			int cs, cur_valign=re->valign, cur_halign=re->halign, cur_posx=-1, cur_posy=-1;
			ssacolors cur_color = re->style->color;
			CGAffineTransform matrix;

			unsigned long inum=0;
			unsigned outputoffset=0, lengthreduce=0;
			BOOL flag=0, newLayout=FALSE, cur_be = FALSE;
			Fixed fixv;

			ATSUCreateAndCopyStyle(re->style->atsustyle,&cur_style);

			%%{
				alphtype unsigned short;

				action bold {SetATSUStyleFlag(cur_style, kATSUQDBoldfaceTag, flag);}

				action italic {SetATSUStyleFlag(cur_style, kATSUQDItalicTag, flag);}

				action uline {SetATSUStyleFlag(cur_style, kATSUQDUnderlineTag, flag);}

				action strike {SetATSUStyleFlag(cur_style, kATSUStyleStrikeThroughTag, flag);}

				action bordersize {cur_outline = num; re->multipart_drawing = YES;}

				action shadowsize {cur_shadow = num; re->multipart_drawing = YES;}

				action bluredge {cur_be = flag;}

				action frot {cur_frz = num; re->multipart_drawing = YES;}

				action fsize {
					num *= (72./96.); // scale from Windows 96dpi
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUSizeTag, sizeof(fixv), &fixv);
				}

				action ftrack {
					fixv = FloatToFixed(num);
					SetATSUStyleOther(cur_style, kATSUTrackingTag, sizeof(fixv), &fixv);
				}

				action fscalex {
					cur_scalex = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action fscaley {
					cur_scaley = num;
					matrix = CGAffineTransformMakeScale(cur_scalex/100.,cur_scaley/100.);
					SetATSUStyleOther(cur_style, kATSUFontMatrixTag, sizeof(matrix), &matrix);
				}

				action strp_begin {
					strparambegin = p;
				}

				action fontname {
					ATSUFontID	font;
					font = FMGetFontFromATSFontRef(ATSFontFindFromName((CFStringRef)[NSString stringWithCharacters:strparambegin length:p-strparambegin],kATSOptionFlagsDefault));
					SetATSUStyleOther(cur_style, kATSUFontTag, sizeof(ATSUFontID), &font);
				}

				action alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(inum, cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action ssa_alignment {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}

					UpdateAlignment(SSA2ASSAlignment(inum), cur_posx, &cur_valign, &cur_halign, cur_layout);
				}

				action pos_begin {
					posbegin=p;
				}

				action pos_end {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}
					NSArray *coo = [[NSString stringWithCharacters:posbegin length:p-posbegin] componentsSeparatedByString:@","];
					cur_posx = [[coo objectAtIndex:0] intValue];
					cur_posy = [[coo objectAtIndex:1] intValue];
				}

				action primarycolor {
					cur_color.primary = ParseColorTag(inum,cur_color.primary.alpha);
					re->multipart_drawing = YES;
				}

				action secondarycolor {
					cur_color.secondary = ParseColorTag(inum,cur_color.secondary.alpha);
					re->multipart_drawing = YES;
				}

				action outlinecolor {
					cur_color.outline = ParseColorTag(inum,cur_color.outline.alpha);
					re->multipart_drawing = YES;
				}

				action shadowcolor {
					cur_color.shadow = ParseColorTag(inum,cur_color.shadow.alpha);
					re->multipart_drawing = YES;
				}

				action primaryalpha {
					cur_color.primary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action secondaryalpha {
					cur_color.secondary.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action outlinealpha {
					cur_color.outline.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action shadowalpha {
					cur_color.shadow.alpha = (255 - inum) / 255.f;
					re->multipart_drawing = YES;
				}

				action nl_handler {
					NSString *append = @"\n";
					parsetmp = [NSString stringWithCharacters:skipbegin length:(p-2)-skipbegin];
					[output appendString:parsetmp];

					skipbegin = p;

					if (*(p - 1) == 'h') append = @" ";

					[output appendString:append];

					lengthreduce++;
				}

				action enter_tag {
					unsigned taglen = p - skipbegin;
					parsetmp = [NSString stringWithCharacters:skipbegin length:taglen];
					[output appendString:parsetmp];
					skipbegin = p;

					cur_range = (NSRange){strbegin - pb, p - strbegin};
					cur_range.location -= outputoffset;
					cur_range.length -= lengthreduce;

					outputoffset += lengthreduce;
					lengthreduce = 0;

                    if (cur_range.length) {
                        [re increasestyles];
                        re->styles[re->style_count-1] = [[SSAStyleSpan alloc] init];
                        re->styles[re->style_count-1]->outline = cur_outline;
                        re->styles[re->style_count-1]->shadow = cur_shadow;
                        re->styles[re->style_count-1]->angle = cur_frz;
                        re->styles[re->style_count-1]->astyle = cur_style;
                        re->styles[re->style_count-1]->range = cur_range;
                        re->styles[re->style_count-1]->outlineblur = cur_be;
                        re->styles[re->style_count-1]->color = cur_color;
                        ATSUCreateAndCopyStyle(cur_style,&cur_style);
                    }
				}

				action exit_tag {
					// XXX exit_tag ideally should run one char later, so we just pretend it does
					p++;
					outputoffset += p - skipbegin;
					skipbegin = p;
					strbegin = p;

					if (newLayout) {
						newLayout = FALSE;
						end_re;

						if ([re->nstext length] != 0) [rentities addObject:re];

						SSARenderEntity *nre = [[[SSARenderEntity alloc] init] autorelease];
						nre->layer = re->layer;
						nre->style = re->style;
						nre->marginl = re->marginl; nre->marginr = re->marginr; nre->marginv = re->marginv; nre->usablewidth = re->usablewidth;
						nre->posx = cur_posx; nre->posy = cur_posy; nre->halign = cur_halign; nre->valign = cur_valign;
						nre->nstext = nil;
						nre->text = nil;
						nre->layout = cur_layout;
						nre->styles = malloc(0);
						nre->style_count = 0;
						nre->disposelayout = YES;
						nre->multipart_drawing = NO;
						nre->is_shape = re->is_shape;
						re = nre;
					}
					p--;
				}

				action color_end {
					NSString *hexn = [NSString stringWithCharacters:intbegin length:p-intbegin];
					inum = strtoul([hexn UTF8String], NULL, 16);
				}

				action styleset {
					if (!newLayout) {
						newLayout = TRUE;
						ATSUCreateAndCopyTextLayout(re->layout,&cur_layout);
					}
					re->multipart_drawing = YES;
					ssastyleline *the_style = re->style;
					NSString *searchsn = [NSString stringWithCharacters:strparambegin length:p-strparambegin];

					if ([searchsn length] > 0) {
						for (j=0; j < ssa->stylecount; j++) {
							if ([searchsn isEqualToString:ssa->styles[j].name])  {
								the_style = &ssa->styles[j];
								break;
							}
						}
					}

					ATSUCopyAttributes(the_style->atsustyle,cur_style);
					cur_color = the_style->color;
					cur_outline = the_style->outline;
					cur_shadow = the_style->shadow;
					cur_frz = the_style->angle;
					cur_scalex = cur_scaley = 1;
					cur_posx = cur_posy = -1;
				}

				action skip_t_tag {
					while (p != pe && *p != ')' && *p != '}') p++;
					p++;
				}

				action draw_mode {
					re->is_shape = inum != 0;
				}

				flag = ([01] % {unichar fl = *(p-1); if (fl == '0' || fl == '1') flag = fl - '0';})? > {flag = 0;};
				num_ = "-"? digit+ ('.' digit*)?;
				num = num_ > {numbegin = p;} % {num = [[NSString stringWithCharacters:numbegin length:p-numbegin] doubleValue];};

				intnum = "-"? (digit+) > {intbegin = p;} % {inum = [[NSString stringWithCharacters:intbegin length:p-intbegin] intValue];};

				color = ("H"|"&"){,2} (xdigit+) > {intbegin = p;} % color_end "&";

				cmd_specific = (("pos(" [^)]+ > pos_begin ")" % pos_end)
								|"bord" num %bordersize
								|"b" flag %bold
								|"be" flag %bluredge
								|"i" flag %italic
								|"u" flag %uline
								|"s" flag %strike
								|"fs" num %fsize
								|"fsp" num %ftrack
								|"fscx" num %fscalex
								|"fscy" num %fscaley
								|("fr" "z"? num %frot)
								|("fn" [^\\}]* > strp_begin %fontname)
								|"shad" num %shadowsize
								|"a" intnum %ssa_alignment
								|"an" intnum %alignment
								|"c" color %primarycolor
								|"1c" color %primarycolor
								|"2c" color %secondarycolor
								|"3c" color %outlinecolor
								|"4c" color %shadowcolor
								|"1a" color %primaryalpha
								|"2a" color %secondaryalpha
								|"3a" color %outlinealpha
								|"4a" color %shadowalpha
								|("r" [^\\}]* > strp_begin %styleset)
								|("fe"|"k"|"kf"|"K"|"ko"|"q"|"fr"|"fad"|"move"|"clip"|"o"|"frx"|"fry") [^\\}]*
								|"p" num %draw_mode
								#|"t" [^)}]* # enabling this crashes ragel
								|"t(" % skip_t_tag
								);

				cmd = ("\\" :> cmd_specific)+;

				plaintext = [^}]*;

				tag = "{" (cmd | plaintext) % exit_tag "}";

				nl = "\\" [Nnh];

				special = nl % nl_handler | (tag > enter_tag);

				text = any*;
				main := (text special?)*;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (pb[len-1] == '}') skipbegin = p; // make up for how exit_tag isn't called if the } is the last char in the line

			end_re;
			if (!cur_range.length) ATSUDisposeStyle(cur_style);
            //PruneIdenticalStyleSpans(re);

			free(pb);
			if ([re->nstext length] != 0) [rentities addObject:re];
		}

	}
	return rentities;
}