Don't require the 2nd Lexer#add_token argument.

%%machine lexer; # %

module Oga
  ##
  # Low level lexer that supports both XML and HTML (using an extra option). To
  # lex HTML input set the `:html` option to `true` when creating an instance
  # of the lexer:
  #
  #     lexer = Oga::Lexer.new(:html => true)
  #
  # @!attribute [r] html
  #  @return [TrueClass|FalseClass]
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    ##
    # Names of the HTML void elements that should be handled when HTML lexing
    # is enabled.
    #
    # @return [Array]
    #
    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    ##
    # @param [Hash] options
    #
    # @option options [Symbol] :html When set to `true` the lexer will treat
    #  the input as HTML instead of SGML/XML. This makes it possible to lex
    #  HTML void elements such as `<link href="">`.
    #
    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    ##
    # Resets the internal state of the lexer. Typically you don't need to call
    # this method yourself as its called by #lex after lexing a given String.
    #
    def reset
      @line     = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @buffer_start_position = nil
    end

    ##
    # Lexes the supplied String and returns an Array of tokens. Each token is
    # an Array in the following format:
    #
    #     [TYPE, VALUE]
    #
    # The type is a symbol, the value is either nil or a String.
    #
    # @param [String] data The string to lex.
    # @return [Array]
    #
    def lex(data)
      @data       = data.unpack('U*')
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    ##
    # @return [TrueClass|FalseClass]
    #
    def html?
      return !!html
    end

    private

    ##
    # @param [Fixnum] amount The amount of lines to advance.
    #
    def advance_line(amount = 1)
      @line += amount
    end

    ##
    # Emits a token who's value is based on the supplied start/stop position.
    #
    # @param [Symbol] type The token type.
    # @param [Fixnum] start
    # @param [Fixnum] stop
    #
    # @see #text
    # @see #add_token
    #
    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    ##
    # Returns the text of the current buffer based on the supplied start and
    # stop position.
    #
    # By default `@ts` and `@te` are used as the start/stop position.
    #
    # @param [Fixnum] start
    # @param [Fixnum] stop
    # @return [String]
    #
    def text(start = @ts, stop = @te)
      return @data[start...stop].pack('U*')
    end

    ##
    # Adds a token with the given type and value to the list.
    #
    # @param [Symbol] type The token type.
    # @param [String] value The token value.
    #
    def add_token(type, value = nil)
      token = [type, value, @line]

      @tokens << token
    end

    ##
    # Enables buffering starting at the given position.
    #
    # @param [Fixnum] position The start position of the buffer, set to `@te`
    #  by default.
    #
    def start_buffer(position = @te)
      @buffer_start_position = position
    end

    ##
    # Returns `true` if we're currently buffering.
    #
    # @return [TrueClass|FalseClass]
    #
    def buffering?
      return !!@buffer_start_position
    end

    ##
    # Emits the current buffer if we have any. The current line number is
    # advanced based on the amount of newlines in the buffer.
    #
    # @param [Fixnum] position The end position of the buffer, set to `@ts` by
    #  default.
    #
    # @param [Symbol] type The type of node to emit.
    #
    def emit_buffer(position = @ts, type = :T_TEXT)
      return unless @buffer_start_position

      content = text(@buffer_start_position, position)

      unless content.empty?
        add_token(type, content)

        lines = content.count("\n")

        advance_line(lines) if lines > 0
      end

      @buffer_start_position = nil
    end

    ##
    # Returns the name of the element we're currently in.
    #
    # @return [String]
    #
    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;
      getkey (@data[p] || 0);

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # Strings
      #
      # Strings in HTML can either be single or double quoted. If a string
      # starts with one of these quotes it must be closed with the same type of
      # quote.
      dquote = '"';
      squote = "'";

      action start_string_dquote {
        start_buffer

        fcall string_dquote;
      }

      action start_string_squote {
        start_buffer

        fcall string_squote;
      }

      # Machine for processing double quoted strings.
      string_dquote := |*
        dquote => {
          emit_buffer(@ts, :T_STRING)
          fret;
        };

        any;
      *|;

      # Machine for processing single quoted strings.
      string_squote := |*
        squote => {
          emit_buffer(@ts, :T_STRING)
          fret;
        };

        any;
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      action start_doctype {
        emit_buffer
        add_token(:T_DOCTYPE_START)
        fcall doctype;
      }

      # Machine for processing doctypes. Doctype values such as the public and
      # system IDs are treated as T_STRING tokens.
      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        '>' => {
          add_token(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      action start_cdata {
        emit_buffer
        add_token(:T_CDATA_START)

        start_buffer

        fcall cdata;
      }

      # Machine that for processing the contents of CDATA tags. Everything
      # inside a CDATA tag is treated as plain text.
      cdata := |*
        cdata_end => {
          emit_buffer
          add_token(:T_CDATA_END)

          fret;
        };

        any;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      action start_comment {
        emit_buffer
        add_token(:T_COMMENT_START)

        start_buffer

        fcall comment;
      }

      # Machine used for processing the contents of a comment. Everything
      # inside a comment is treated as plain text (similar to CDATA tags).
      comment := |*
        comment_end => {
          emit_buffer
          add_token(:T_COMMENT_END)

          fret;
        };

        any;
      *|;

      # XML declaration tags
      #
      # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
      #
      xml_decl_start = '<?xml';
      xml_decl_end   = '?>';

      action start_xml_decl {
        emit_buffer
        add_token(:T_XML_DECL_START)

        start_buffer

        fcall xml_decl;
      }

      # Machine that processes the contents of an XML declaration tag.
      xml_decl := |*
        xml_decl_end => {
          emit_buffer
          add_token(:T_XML_DECL_END)

          fret;
        };

        any;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action start_element {
        emit_buffer
        add_token(:T_ELEM_START)

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element_head;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      # Machine used for processing the characters inside a element head. An
      # element head is everything between `<NAME` (where NAME is the element
      # name) and `>`.
      #
      # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
      #
      element_head := |*
        whitespace | '=';

        newline => { advance_line };

        # Attribute names.
        element_name => { t(:T_ATTR) };

        # Attribute values.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # The closing character of the open tag.
        ('>' | '/') => {
          fhold;
          fret;
        };
      *|;

      main := |*
        element_start  => start_element;
        doctype_start  => start_doctype;
        cdata_start    => start_cdata;
        comment_start  => start_comment;
        xml_decl_start => start_xml_decl;

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_END, nil)
            @elements.pop
          end
        };

        # Regular closing tags.
        '</' element_name '>' => {
          emit_buffer
          add_token(:T_ELEM_END, nil)

          @elements.pop
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
          add_token(:T_ELEM_END, nil)

          @elements.pop
        };

        # Note that this rule should be declared at the very bottom as it will
        # otherwise take precedence over the other rules.
        any => {
          # First character, start buffering (unless we already are buffering).
          start_buffer(@ts) unless buffering?

          # EOF, emit the text buffer.
          if @te == eof
            emit_buffer(@te)
          end
        };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  # Low level lexer that supports both XML and HTML (using an extra option). To
  # lex HTML input set the `:html` option to `true` when creating an instance
  # of the lexer:
  #
  #     lexer = Oga::Lexer.new(:html => true)
  #
  # @!attribute [r] html
  #  @return [TrueClass|FalseClass]
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    ##
    # Names of the HTML void elements that should be handled when HTML lexing
    # is enabled.
    #
    # @return [Array]
    #
    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    ##
    # @param [Hash] options
    #
    # @option options [Symbol] :html When set to `true` the lexer will treat
    #  the input as HTML instead of SGML/XML. This makes it possible to lex
    #  HTML void elements such as `<link href="">`.
    #
    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    ##
    # Resets the internal state of the lexer. Typically you don't need to call
    # this method yourself as its called by #lex after lexing a given String.
    #
    def reset
      @line     = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @buffer_start_position = nil
    end

    ##
    # Lexes the supplied String and returns an Array of tokens. Each token is
    # an Array in the following format:
    #
    #     [TYPE, VALUE]
    #
    # The type is a symbol, the value is either nil or a String.
    #
    # @param [String] data The string to lex.
    # @return [Array]
    #
    def lex(data)
      @data       = data.unpack('U*')
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    ##
    # @return [TrueClass|FalseClass]
    #
    def html?
      return !!html
    end

    private

    ##
    # @param [Fixnum] amount The amount of lines to advance.
    #
    def advance_line(amount = 1)
      @line += amount
    end

    ##
    # Emits a token who's value is based on the supplied start/stop position.
    #
    # @param [Symbol] type The token type.
    # @param [Fixnum] start
    # @param [Fixnum] stop
    #
    # @see #text
    # @see #add_token
    #
    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    ##
    # Returns the text of the current buffer based on the supplied start and
    # stop position.
    #
    # By default `@ts` and `@te` are used as the start/stop position.
    #
    # @param [Fixnum] start
    # @param [Fixnum] stop
    # @return [String]
    #
    def text(start = @ts, stop = @te)
      return @data[start...stop].pack('U*')
    end

    ##
    # Adds a token with the given type and value to the list.
    #
    # @param [Symbol] type The token type.
    # @param [String] value The token value.
    #
    def add_token(type, value)
      token = [type, value, @line]

      @tokens << token
    end

    ##
    # Enables buffering starting at the given position.
    #
    # @param [Fixnum] position The start position of the buffer, set to `@te`
    #  by default.
    #
    def start_buffer(position = @te)
      @buffer_start_position = position
    end

    ##
    # Returns `true` if we're currently buffering.
    #
    # @return [TrueClass|FalseClass]
    #
    def buffering?
      return !!@buffer_start_position
    end

    ##
    # Emits the current buffer if we have any. The current line number is
    # advanced based on the amount of newlines in the buffer.
    #
    # @param [Fixnum] position The end position of the buffer, set to `@ts` by
    #  default.
    #
    # @param [Symbol] type The type of node to emit.
    #
    def emit_buffer(position = @ts, type = :T_TEXT)
      return unless @buffer_start_position

      content = text(@buffer_start_position, position)

      unless content.empty?
        add_token(type, content)

        lines = content.count("\n")

        advance_line(lines) if lines > 0
      end

      @buffer_start_position = nil
    end

    ##
    # Returns the name of the element we're currently in.
    #
    # @return [String]
    #
    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;
      getkey (@data[p] || 0);

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # Strings
      #
      # Strings in HTML can either be single or double quoted. If a string
      # starts with one of these quotes it must be closed with the same type of
      # quote.
      dquote = '"';
      squote = "'";

      action start_string_dquote {
        start_buffer

        fcall string_dquote;
      }

      action start_string_squote {
        start_buffer

        fcall string_squote;
      }

      # Machine for processing double quoted strings.
      string_dquote := |*
        dquote => {
          emit_buffer(@ts, :T_STRING)
          fret;
        };

        any;
      *|;

      # Machine for processing single quoted strings.
      string_squote := |*
        squote => {
          emit_buffer(@ts, :T_STRING)
          fret;
        };

        any;
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      action start_doctype {
        emit_buffer
        add_token(:T_DOCTYPE_START, nil)
        fcall doctype;
      }

      # Machine for processing doctypes. Doctype values such as the public and
      # system IDs are treated as T_STRING tokens.
      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        '>' => {
          add_token(:T_DOCTYPE_END, nil)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      action start_cdata {
        emit_buffer
        add_token(:T_CDATA_START, nil)

        start_buffer

        fcall cdata;
      }

      # Machine that for processing the contents of CDATA tags. Everything
      # inside a CDATA tag is treated as plain text.
      cdata := |*
        cdata_end => {
          emit_buffer
          add_token(:T_CDATA_END, nil)

          fret;
        };

        any;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      action start_comment {
        emit_buffer
        add_token(:T_COMMENT_START, nil)

        start_buffer

        fcall comment;
      }

      # Machine used for processing the contents of a comment. Everything
      # inside a comment is treated as plain text (similar to CDATA tags).
      comment := |*
        comment_end => {
          emit_buffer
          add_token(:T_COMMENT_END, nil)

          fret;
        };

        any;
      *|;

      # XML declaration tags
      #
      # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
      #
      xml_decl_start = '<?xml';
      xml_decl_end   = '?>';

      action start_xml_decl {
        emit_buffer
        add_token(:T_XML_DECL_START, nil)

        start_buffer

        fcall xml_decl;
      }

      # Machine that processes the contents of an XML declaration tag.
      xml_decl := |*
        xml_decl_end => {
          emit_buffer
          add_token(:T_XML_DECL_END, nil)

          fret;
        };

        any;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action start_element {
        emit_buffer
        add_token(:T_ELEM_START, nil)

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element_head;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      # Machine used for processing the characters inside a element head. An
      # element head is everything between `<NAME` (where NAME is the element
      # name) and `>`.
      #
      # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
      #
      element_head := |*
        whitespace | '=';

        newline => { advance_line };

        # Attribute names.
        element_name => { t(:T_ATTR) };

        # Attribute values.
        dquote => start_string_dquote;
        squote => start_string_squote;

        # The closing character of the open tag.
        ('>' | '/') => {
          fhold;
          fret;
        };
      *|;

      main := |*
        element_start  => start_element;
        doctype_start  => start_doctype;
        cdata_start    => start_cdata;
        comment_start  => start_comment;
        xml_decl_start => start_xml_decl;

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_END, nil)
            @elements.pop
          end
        };

        # Regular closing tags.
        '</' element_name '>' => {
          emit_buffer
          add_token(:T_ELEM_END, nil)

          @elements.pop
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
          add_token(:T_ELEM_END, nil)

          @elements.pop
        };

        # Note that this rule should be declared at the very bottom as it will
        # otherwise take precedence over the other rules.
        any => {
          # First character, start buffering (unless we already are buffering).
          start_buffer(@ts) unless buffering?

          # EOF, emit the text buffer.
          if @te == eof
            emit_buffer(@te)
          end
        };
      *|;
    }%%
  end # Lexer
end # Oga


Added more escapes and members to sets (meta characters and utf8)

%%{
  machine re_scanner;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  # Property names are being treated as case-insensitive, but it is not clear
  # yet if this applies to all flavors and in all encodings. A bug has just
  # been filed against ruby regarding this issue.
  # http://redmine.ruby-lang.org/issues/show/4014
  property_name_unicode = 'alnum'i | 'alpha'i | 'any'i   | 'ascii'i | 'blank'i |
                          'cntrl'i | 'digit'i | 'graph'i | 'lower'i | 'print'i |
                          'punct'i | 'space'i | 'upper'i | 'word'i  | 'xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha; # FIXME: incorrect, and can include escapes

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  # group (nesting) and set open/close actions
  action group_opened {
    #puts "GROUP: OPENED"
    in_group += 1
  }

  action group_closed {
    #puts "GROUP: CLOSED"
    in_group -= 1
  }

  action set_opened {
    #puts "SET: OPENED"
    in_set = true
  }

  action set_closed {
    #puts "SET: CLOSED"
    in_set = false
  }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    meta_char {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    ascii_print    |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts-1, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    meta_char {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    ascii_print    |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # FIXME: a backspace only inside a set
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts-1..te-1].pack('c*')

      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
      fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open %set_opened {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options %group_opened {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type %group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type %group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open %group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close %group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    backslash . range_open . (digit+)? . ','? . (digit+)? . backslash . range_close {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returns at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group = 0
    in_set   = false

    %% write init;
    %% write exec;

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join
    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  # Property names are being treated as case-insensitive, but it is not clear
  # yet if this applies to all flavors and in all encodings. A bug has just
  # been filed against ruby regarding this issue.
  # http://redmine.ruby-lang.org/issues/show/4014
  property_name_unicode = 'alnum'i | 'alpha'i | 'any'i   | 'ascii'i | 'blank'i |
                          'cntrl'i | 'digit'i | 'graph'i | 'lower'i | 'print'i |
                          'punct'i | 'space'i | 'upper'i | 'word'i  | 'xdigit'i;

  property_name_ruby    = 'any'i | 'assigned'i | 'newline'i;

  property_name         = property_name_unicode | property_name_ruby;

  category_letter       = [Ll] . [ultmo]?;
  category_mark         = [Mm] . [nce]?;
  category_number       = [Nn] . [dlo]?;
  category_punctuation  = [Pp] . [cdseifo]?;
  category_symbol       = [Ss] . [mcko]?;
  category_separator    = [Zz] . [slp]?;
  category_codepoint    = [Cc] . [cfson]?;

  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  class_posix           = '[:' . class_name_posix . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . alpha;
  meta_sequence         = 'M-' . alpha; # FIXME: incorrect, and can include escapes

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':'?;

  group_name            = alpha . alnum+;
  group_named           = ('?<' . group_name . '>') | ('?\'' . group_name . '\'');

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  # group (nesting) and set open/close actions
  action group_opened {
    #puts "GROUP: OPENED"
    in_group += 1
  }

  action group_closed {
    #puts "GROUP: CLOSED"
    in_group -= 1
  }

  action set_opened {
    #puts "SET: OPENED"
    in_set = true
  }

  action set_closed {
    #puts "SET: CLOSED"
    in_set = false
  }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' %set_closed {
      self.emit(:set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '-]' %set_closed { # special case, emits two tokens
      self.emit(:set, :member, data[ts..te-2].pack('c*'), ts, te)
      self.emit(:set, :close,  data[ts+1..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      text = data[ts..te-1].pack('c*')
      if @tokens.last[1] == :open
        self.emit(:set, :negate, text, ts, te)
      else
        self.emit(:set, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' {
      fcall set_escape_sequence;
    };

    class_posix {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:set, :class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:set, :class_alpha,  text, ts, te)
      when '[:ascii:]';  self.emit(:set, :class_ascii,  text, ts, te)
      when '[:blank:]';  self.emit(:set, :class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:set, :class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:set, :class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:set, :class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:set, :class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:set, :class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:set, :class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:set, :class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:set, :class_upper,  text, ts, te)
      when '[:word:]';   self.emit(:set, :class_word,   text, ts, te)
      when '[:xdigit:]'; self.emit(:set, :class_xdigit, text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' {
      self.emit(:set, :backspace, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [\\\]\-\,] {
      self.emit(:set, :escape, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    [dDhHsSwW] {
      case text = data[ts-1..te-1].pack('c*')
      when '\d'; self.emit(:set, :type_digit,     text, ts-1, te)
      when '\D'; self.emit(:set, :type_nondigit,  text, ts-1, te)
      when '\h'; self.emit(:set, :type_hex,       text, ts-1, te)
      when '\H'; self.emit(:set, :type_nonhex,    text, ts-1, te)
      when '\s'; self.emit(:set, :type_space,     text, ts-1, te)
      when '\S'; self.emit(:set, :type_nonspace,  text, ts-1, te)
      when '\w'; self.emit(:set, :type_word,      text, ts-1, te)
      when '\W'; self.emit(:set, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      self.emit(:set, :range_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };

    hex_sequence {
      self.emit(:set, :member_hex, data[ts-1..te-1].pack('c*'), ts-1, te)
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = data[ts-1..te-1].pack('c*')
      self.emit(:backref, :digit, text, ts, te)
      fret;
    };

    meta_char {
      case text = data[ts-1..te-1].pack('c*')
      when '\.';  self.emit(:escape, :dot,               text, ts-1, te)
      when '\|';  self.emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  self.emit(:escape, :beginning_of_line, text, ts-1, te)
      when '\$';  self.emit(:escape, :end_of_line,       text, ts-1, te)
      when '\?';  self.emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  self.emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  self.emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  self.emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  self.emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  self.emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  self.emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  self.emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  self.emit(:escape, :set_close,         text, ts-1, te)
      when '\\\\';
        self.emit(:escape, :backslash, text, ts, te)
      end
      fret;
    };

    escaped_char > (escaped_alpha, 8) {
      case text = data[ts-1..te-1].pack('c*')
      when '\a'; self.emit(:escape, :bell,           text, ts, te)
      when '\b'; self.emit(:escape, :backspace,      text, ts, te) # FIXME: a backspace only inside a set
      when '\e'; self.emit(:escape, :escape,         text, ts, te)
      when '\f'; self.emit(:escape, :form_feed,      text, ts, te)
      when '\n'; self.emit(:escape, :newline,        text, ts, te)
      when '\r'; self.emit(:escape, :carriage,       text, ts, te)
      when '\s'; self.emit(:escape, :space,          text, ts, te)
      when '\t'; self.emit(:escape, :tab,            text, ts, te)
      when '\v'; self.emit(:escape, :vertical_tab,   text, ts, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 7) {
      text = data[ts-1..te-1].pack('c*')
      if text[2].chr == '{'
        self.emit(:escape, :codepoint_list, text, ts, te)
      else
        self.emit(:escape, :codepoint,      text, ts, te)
      end
      fret;
    };

    octal_sequence {
      self.emit(:escape, :octal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    hex_sequence > (escaped_alpha, 6) {
      self.emit(:escape, :hex, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # FIXME: scanner returns nil
    wide_hex_sequence > (escaped_alpha, 5) {
      self.emit(:escape, :hex_wide, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence > (escaped_alpha, 4) {
      self.emit(:escape, :control, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence > (escaped_alpha, 3) {
      # TODO: add escapes
      self.emit(:escape, :meta, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts-1..te-1].pack('c*')

      type = text[1,1] == 'p' ? :property : :nonproperty
      # TODO: add ^ for property negation, :nonproperty_caret

      case name = data[ts+2..te-2].pack('c*').downcase

      # Named
      when 'alnum';   self.emit(type, :alnum,       text, ts, te)
      when 'alpha';   self.emit(type, :alpha,       text, ts, te)
      when 'any';     self.emit(type, :any,         text, ts, te)
      when 'ascii';   self.emit(type, :ascii,       text, ts, te)
      when 'blank';   self.emit(type, :blank,       text, ts, te)
      when 'cntrl';   self.emit(type, :cntrl,       text, ts, te)
      when 'digit';   self.emit(type, :digit,       text, ts, te)
      when 'graph';   self.emit(type, :graph,       text, ts, te)
      when 'lower';   self.emit(type, :lower,       text, ts, te)
      when 'newline'; self.emit(type, :newline,     text, ts, te)
      when 'print';   self.emit(type, :print,       text, ts, te)
      when 'punct';   self.emit(type, :punct,       text, ts, te)
      when 'space';   self.emit(type, :space,       text, ts, te)
      when 'upper';   self.emit(type, :upper,       text, ts, te)
      when 'word';    self.emit(type, :word,        text, ts, te)
      when 'xdigit';  self.emit(type, :xdigit,      text, ts, te)

      # Letters
      when 'l';  self.emit(type, :letter_any,       text, ts, te)
      when 'lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'm';  self.emit(type, :mark_any,         text, ts, te)
      when 'mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'n';  self.emit(type, :number_any,       text, ts, te)
      when 'nd'; self.emit(type, :number_decimal,   text, ts, te)
      when 'nl'; self.emit(type, :number_letter,    text, ts, te)
      when 'no'; self.emit(type, :number_other,     text, ts, te)

      # Punctuation
      when 'p';  self.emit(type, :punct_any,        text, ts, te)
      when 'pc'; self.emit(type, :punct_connector,  text, ts, te)
      when 'pd'; self.emit(type, :punct_dash,       text, ts, te)
      when 'ps'; self.emit(type, :punct_open,       text, ts, te)
      when 'pe'; self.emit(type, :punct_close,      text, ts, te)
      when 'pi'; self.emit(type, :punct_initial,    text, ts, te)
      when 'pf'; self.emit(type, :punct_final,      text, ts, te)
      when 'po'; self.emit(type, :punct_other,      text, ts, te)

      # Symbols
      when 's';  self.emit(type, :symbol_any,       text, ts, te)
      when 'sm'; self.emit(type, :symbol_math,      text, ts, te)
      when 'sc'; self.emit(type, :symbol_currency,  text, ts, te)
      when 'sk'; self.emit(type, :symbol_modifier,  text, ts, te)
      when 'so'; self.emit(type, :symbol_other,     text, ts, te)

      # Separators
      when 'z';  self.emit(type, :separator_any,    text, ts, te)
      when 'zs'; self.emit(type, :separator_space,  text, ts, te)
      when 'zl'; self.emit(type, :separator_line,   text, ts, te)
      when 'zp'; self.emit(type, :separator_para,   text, ts, te)

      # Codepoints
      when 'c';  self.emit(type, :cp_any,           text, ts, te)
      when 'cc'; self.emit(type, :cp_control,       text, ts, te)
      when 'cf'; self.emit(type, :cp_format,        text, ts, te)
      when 'cs'; self.emit(type, :cp_surrogate,     text, ts, te)
      when 'co'; self.emit(type, :cp_private,       text, ts, te)
      when 'cn'; self.emit(type, :cp_unassigned,    text, ts, te)
      end
      fret;
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape, :literal, data[ts-1..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    dot {
      self.emit(:meta, :dot, data[ts..te-1].pack('c*'), ts, te)
    };

    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:type, :digit,      text, ts, te)
      when '\\D'; self.emit(:type, :nondigit,   text, ts, te)
      when '\\h'; self.emit(:type, :hex,        text, ts, te)
      when '\\H'; self.emit(:type, :nonhex,     text, ts, te)
      when '\\s'; self.emit(:type, :space,      text, ts, te)
      when '\\S'; self.emit(:type, :nonspace,   text, ts, te)
      when '\\w'; self.emit(:type, :word,       text, ts, te)
      when '\\W'; self.emit(:type, :nonword,    text, ts, te)
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :nonword_boundary,   text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open %set_opened {
      self.emit(:set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options %group_opened {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type %group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type %group_opened {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)

      when /\(\?<\w+>/
        self.emit(:group, :named,     text, ts, te)
      when /\(\?'\w+'/
        self.emit(:group, :named_sq,  text, ts, te)
      end
    };

    group_open %group_opened {
      text =  data[ts..te-1].pack('c*')
      self.emit(:group, :capture, text, ts, te)
    };

    group_close %group_closed {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # BRE version
    backslash . range_open . (digit+)? . ','? . (digit+)? . backslash . range_close {
      self.emit(:quantifier, :interval_bre, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      self.append_literal(data, ts, te)
    };

  *|;
}%%

module Regexp::Scanner
  %% write data;

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returns at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This may raise an error if a syntax error is encountered. ** this is still
  # in progress.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group = 0
    in_set   = false

    %% write init;
    %% write exec;

    # when the entire expression is a literal run
    self.emit_literal if @literal

    @tokens
  end

  # appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. contents a mix of ASCII and UTF-8
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << [data[ts..te-1].pack('c*'), ts, te]
  end

  # emits the collected literal run collected by one or more calls to the
  # append_literal method
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join
    self.emit(:literal, :literal, text, ts, te)
    @literal = nil
  end

  def self.emit(type, token, text, ts, te)
    #puts " > emit: #{type}:#{token} '#{text}' [#{ts}..#{te}]"

    if @literal and type != :literal
      self.emit_literal
    end

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

end # module Regexp::Scanner


Fixed 512-byte checking in the parser

Fixed 512-byte checking in the parser

package me.parted.anivia.irc;

import org.jetbrains.annotations.Nullable;

import java.util.ArrayList;
import java.util.List;

/**
 * The {@code IrcMessageReader} decodes a {@code String} containing the IRC protocol message as described in RFC 2812
 * Section 2.3 and in the IRCv3.2 specification into an {@link me.parted.anivia.irc.IrcMessage}.
 * <p>
 * The code in this class is generated through Ragel, a finite state machine compiler that compiles executable finite
 * state machines from regular languages. As a result, this code is not maintainable except from the Ragel-based input
 * file that contains rules on how to parse the protocol.
 *
 * @author Justin Kaufman
 * @see me.parted.anivia.irc.IrcMessage
 * @since 1.0
 */
public class IrcMessageReader {

    /**
     * Decodes a {@code String} containing a raw IRC protocol message as described in RFC 2812 Section 2.3 and in the
     * IRCv3.2 specification.
     * @param raw The raw line to be processed.
     * @return An {@code IrcMessage} containing the data processed from the raw line.
     */
    @Nullable
    public static IrcMessage read(String raw) {
        if (raw == null) {
            return null;
        }

        if (raw.length() > 1024) {
            return null;
        }
        /* Construction variables */
        IrcMessage ircMessage = null;

        List<IrcTag> ircTags = new ArrayList<>();
        IrcTag ircTag = null;
        String tagKey = null;
        String tagValue = null;

        IrcPrefix ircPrefix = null;
        String hostname = null;
        String nickname = null;
        String user = null;
        String host = null;

        String command = null;

        String params = null;

        /* Required tokens for the Ragel FSM */
        char[] data = raw.toCharArray();
        int cs, p = 0, pe = data.length, eof = data.length;
        int s = p; // current index
		int q = p; // section start

        %%{
            machine irc;

			action sectionstart { q = p; }
            action bufferstart { s = p; }
            action checksize {
                if (p - q > 512) {
                    return null;
                }
            }
            action tagkey {
                tagKey = new String(data, s, p - s);
            }
            action tagvalue {
                tagValue = new String(data, s, p - s);
            }
            action tag {
                ircTag = new IrcTag(tagKey, tagValue);
                ircTags.add(ircTag);
                tagValue = null; // the next tag could have a null value
            }
            action hostname {
                hostname = new String(data, s, p - s);
            }
            action host {
                host = new String(data, s, p - s);
            }
            action nickname {
                nickname = new String(data, s, p - s);
            }
            action user {
                user = new String(data, s, p - s);
            }
            action prefix {
                if (nickname == null) {
                    ircPrefix = new IrcPrefix(hostname);
                } else {
                    ircPrefix = new IrcPrefix(nickname, user, host);
                }
            }
            action command {
                command = new String(data, s, p - s);
            }
            action params {
                params = new String(data, s, p - s);
            }
            action message {
                ircMessage = new IrcMessage(ircTags, ircPrefix, command, params);
            }

            decoctet    = digit
                | ( ('1'..'9') digit )
                | ( '1' digit digit )
                | ( '2' ('0'..'4') digit )
                | ( '25' ('0'..'5') );
            ip4addr     = decoctet '.' decoctet '.' decoctet '.' decoctet;
            h16         = xdigit{1,4};
            ls32        = ( h16 ":" h16 ) | ip4addr;
            ip6addr     =                   ( ( h16 ':' ){6} ls32 )
                |                      ( '::' ( h16 ':' ){5} ls32 )
                |             ( ( h16 )? '::' ( h16 ':' ){4} ls32 )
                | ( ((h16 ':'){,1} h16)? '::' ( h16 ':' ){3} ls32 )
                | ( ((h16 ':'){,2} h16)? '::' ( h16 ':' ){2} ls32 )
                | ( ((h16 ':'){,3} h16)? '::'   h16 ':'      ls32 )
                | ( ((h16 ':'){,4} h16)? '::'                ls32 )
                | ( ((h16 ':'){,5} h16)? '::'                h16  )
                | ( ((h16 ':'){,6} h16)? '::'                     );
            hostaddr   = ip4addr | ip6addr;
            shortname  = alnum ( (alnum | '-')* alnum )?;
            hostname   = ( shortname ('.' shortname)* ) >bufferstart %hostname;
            host       = ( hostname | hostaddr ) >bufferstart %host;
            vendor     = hostname;
            tagvalue   = (extend - [\0\r\n; ])+ >bufferstart %tagvalue;
            tagkey     = ( (vendor '/')? (alpha | digit | '-')+ ) >bufferstart %tagkey;
            tag        = ( tagkey ('=' tagvalue)? ) %tag;
            tags       = ( tag ( ';' tag )* );
            user       = ( extend - [\0\r\n @] )+ >bufferstart %user;
            special    = ( 0x5B..0x60 ) | ( 0x7B..0x7D );
            nickname   = ( (alpha | special) (alnum | special | '-' ){,29} ) >bufferstart %nickname;
            prefix     = ( hostname | (nickname (("!" user)? "@" host)?) ) %prefix;
            command    = ( alpha+ | digit{3} ) >bufferstart %command;
            nospcrlfcl = extend - [\0\r\n :];
            middle     = nospcrlfcl ( ':' | nospcrlfcl )*;
            trailing   = ( ':' | ' ' | nospcrlfcl )*;
            params     = ( (' ' middle){,14} (' :' trailing)? ) >bufferstart %params;
            crlf       = '\r\n';
            message1   = ( '@' tags ' ' )? >sectionstart $checksize;
            message2   = ( ':' prefix ' ' )? command ( params )? crlf >sectionstart $checksize;
            message    = message1 message2;
            main      := message %message;

            write init;
            write exec;
        }%%

        return ircMessage;
    }

    %% write data;

}

package me.parted.anivia.irc;

import org.jetbrains.annotations.Nullable;

import java.util.ArrayList;
import java.util.List;

/**
 * The {@code IrcMessageReader} decodes a {@code String} containing the IRC protocol message as described in RFC 2812
 * Section 2.3 and in the IRCv3.2 specification into an {@link me.parted.anivia.irc.IrcMessage}.
 * <p>
 * The code in this class is generated through Ragel, a finite state machine compiler that compiles executable finite
 * state machines from regular languages. As a result, this code is not maintainable except from the Ragel-based input
 * file that contains rules on how to parse the protocol.
 *
 * @author Justin Kaufman
 * @see me.parted.anivia.irc.IrcMessage
 * @since 1.0
 */
public class IrcMessageReader {

    /**
     * Decodes a {@code String} containing a raw IRC protocol message as described in RFC 2812 Section 2.3 and in the
     * IRCv3.2 specification.
     * @param raw The raw line to be processed.
     * @return An {@code IrcMessage} containing the data processed from the raw line.
     */
    @Nullable
    public static IrcMessage read(String raw) {
        if (raw == null) {
            return null;
        }

        if (raw.length() > 1024) {
            return null;
        }
        /* Construction variables */
        IrcMessage ircMessage = null;

        List<IrcTag> ircTags = new ArrayList<>();
        IrcTag ircTag = null;
        String tagKey = null;
        String tagValue = null;

        IrcPrefix ircPrefix = null;
        String hostname = null;
        String nickname = null;
        String user = null;
        String host = null;

        String command = null;

        String params = null;

        /* Required tokens for the Ragel FSM */
        char[] data = raw.toCharArray();
        int cs, p = 0, pe = data.length, eof = data.length;
        int s = p; // current index

        %%{
            machine irc;

            action bufferstart { s = p; }
            action checksize {
                if (p - s > 512) {
                    return null;
                }
            }
            action tagkey {
                tagKey = new String(data, s, p - s);
            }
            action tagvalue {
                tagValue = new String(data, s, p - s);
            }
            action tag {
                ircTag = new IrcTag(tagKey, tagValue);
                ircTags.add(ircTag);
                tagValue = null; // the next tag could have a null value
            }
            action hostname {
                hostname = new String(data, s, p - s);
            }
            action host {
                host = new String(data, s, p - s);
            }
            action nickname {
                nickname = new String(data, s, p - s);
            }
            action user {
                user = new String(data, s, p - s);
            }
            action prefix {
                if (nickname == null) {
                    ircPrefix = new IrcPrefix(hostname);
                } else {
                    ircPrefix = new IrcPrefix(nickname, user, host);
                }
            }
            action command {
                command = new String(data, s, p - s);
            }
            action params {
                params = new String(data, s, p - s);
            }
            action message {
                ircMessage = new IrcMessage(ircTags, ircPrefix, command, params);
            }

            decoctet    = digit
                | ( ('1'..'9') digit )
                | ( '1' digit digit )
                | ( '2' ('0'..'4') digit )
                | ( '25' ('0'..'5') );
            ip4addr     = decoctet '.' decoctet '.' decoctet '.' decoctet;
            h16         = xdigit{1,4};
            ls32        = ( h16 ":" h16 ) | ip4addr;
            ip6addr     =                   ( ( h16 ':' ){6} ls32 )
                |                      ( '::' ( h16 ':' ){5} ls32 )
                |             ( ( h16 )? '::' ( h16 ':' ){4} ls32 )
                | ( ((h16 ':'){,1} h16)? '::' ( h16 ':' ){3} ls32 )
                | ( ((h16 ':'){,2} h16)? '::' ( h16 ':' ){2} ls32 )
                | ( ((h16 ':'){,3} h16)? '::'   h16 ':'      ls32 )
                | ( ((h16 ':'){,4} h16)? '::'                ls32 )
                | ( ((h16 ':'){,5} h16)? '::'                h16  )
                | ( ((h16 ':'){,6} h16)? '::'                     );
            hostaddr   = ip4addr | ip6addr;
            shortname  = alnum ( (alnum | '-')* alnum )?;
            hostname   = ( shortname ('.' shortname)* ) >bufferstart %hostname;
            host       = ( hostname | hostaddr ) >bufferstart %host;
            vendor     = hostname;
            tagvalue   = (extend - [\0\r\n; ])+ >bufferstart %tagvalue;
            tagkey     = ( (vendor '/')? (alpha | digit | '-')+ ) >bufferstart %tagkey;
            tag        = ( tagkey ('=' tagvalue)? ) %tag;
            tags       = ( tag ( ';' tag )* );
            user       = ( extend - [\0\r\n @] )+ >bufferstart %user;
            special    = ( 0x5B..0x60 ) | ( 0x7B..0x7D );
            nickname   = ( (alpha | special) (alnum | special | '-' ){,29} ) >bufferstart %nickname;
            prefix     = ( hostname | (nickname (("!" user)? "@" host)?) ) %prefix;
            command    = ( alpha+ | digit{3} ) >bufferstart %command;
            nospcrlfcl = extend - [\0\r\n :];
            middle     = nospcrlfcl ( ':' | nospcrlfcl )*;
            trailing   = ( ':' | ' ' | nospcrlfcl )*;
            params     = ( (' ' middle){,14} (' :' trailing)? ) >bufferstart %params;
            crlf       = '\r\n';
            message1   = ( '@' tags ' ' )? >bufferstart %checksize;
            message2   = ( ':' prefix ' ' )? command ( params )? crlf >bufferstart %checksize;
            message    = message1 message2;
            main      := message %message;

            write init;
            write exec;
        }%%

        return ircMessage;
    }

    %% write data;

}


Update lexer.rl

Update lexer.rl

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = @p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( number         >mark %{ emit :number     } ) |
           ( string_double  >mark %{ emit :dstring    } ) |
           ( string_single  >mark %{ emit :sstring    } ) |
           ( istring_start  >mark %{ emit :istart     } ) |
           ( istring_middle >mark %{ emit :imiddle    } ) |
           ( istring_end    >mark %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( unops          >mark %{ emit :unop       } ) |
           ( binops         >mark %{ emit :binop      } ) |
           ( keywords       >mark %{ emit :keyword    } ) |
           ( identifier     >mark %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( number         >mark %{ emit :number     } ) |
           ( string_double  >mark %{ emit :dstring    } ) |
           ( string_single  >mark %{ emit :sstring    } ) |
           ( istring_start  >mark %{ emit :istart     } ) |
           ( istring_middle >mark %{ emit :imiddle    } ) |
           ( istring_end    >mark %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( unops          >mark %{ emit :unop       } ) |
           ( binops         >mark %{ emit :binop      } ) |
           ( keywords       >mark %{ emit :keyword    } ) |
           ( identifier     >mark %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end


[jancy] '\n' token in doxy-lexer had incorrect line number (line was incremented before the token was created). it lead to premature termination of \brief block on the very first \n

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace jnc {
namespace ct {

//.............................................................................

%%{

machine dox;
write data;

# . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# definitions
#

# regular char (non-whitespace and non-escape)

rc = [^ \t\r\n\\];
ws = [ \t\r]+;

# . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# main machine
#

main := |*

'\\enum'          { createToken (DoxyTokenKind_Enum); };
'\\struct'        { createToken (DoxyTokenKind_Struct); };
'\\union'         { createToken (DoxyTokenKind_Union); };
'\\class'         { createToken (DoxyTokenKind_Class); };
'\\fn'            { createToken (DoxyTokenKind_Fn); };
'\\group'         { createToken (DoxyTokenKind_Group); };
'\\defgroup'      { createToken (DoxyTokenKind_DefGroup); };
'\\ingroup'       { createToken (DoxyTokenKind_InGroup); };
'\\addtogroup'    { createToken (DoxyTokenKind_AddToGroup); };
'\\title'         { createToken (DoxyTokenKind_Title); };
'\\brief'         { createToken (DoxyTokenKind_Brief); };

'\\' rc*          { createToken (DoxyTokenKind_OtherCommand); };

'@{'              { createToken (DoxyTokenKind_OpeningBrace); };
'@}'              { createToken (DoxyTokenKind_ClosingBrace); };

rc ([^\n\\]* rc)? { createTextToken (); };

'\n'              { createToken ('\n'), newLine (ts + 1); };
ws                ;

*|;

}%%

//.............................................................................

void
DoxyLexer::init ()
{
	%% write init;
}

void
DoxyLexer::exec ()
{
	%% write exec;
}

//.............................................................................

} // namespace ct
} // namespace jnc

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace jnc {
namespace ct {

//.............................................................................

%%{

machine dox;
write data;

# . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# definitions
#

# regular char (non-whitespace and non-escape)

rc = [^ \t\r\n\\];
ws = [ \t\r]+;

# . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# main machine
#

main := |*

'\\enum'          { createToken (DoxyTokenKind_Enum); };
'\\struct'        { createToken (DoxyTokenKind_Struct); };
'\\union'         { createToken (DoxyTokenKind_Union); };
'\\class'         { createToken (DoxyTokenKind_Class); };
'\\fn'            { createToken (DoxyTokenKind_Fn); };
'\\group'         { createToken (DoxyTokenKind_Group); };
'\\defgroup'      { createToken (DoxyTokenKind_DefGroup); };
'\\ingroup'       { createToken (DoxyTokenKind_InGroup); };
'\\addtogroup'    { createToken (DoxyTokenKind_AddToGroup); };
'\\title'         { createToken (DoxyTokenKind_Title); };
'\\brief'         { createToken (DoxyTokenKind_Brief); };

'\\' rc*          { createToken (DoxyTokenKind_OtherCommand); };

'@{'              { createToken (DoxyTokenKind_OpeningBrace); };
'@}'              { createToken (DoxyTokenKind_ClosingBrace); };

rc ([^\n\\]* rc)? { createTextToken (); };

'\n'              { newLine (ts + 1), createToken ('\n'); };
ws                ;

*|;

}%%

//.............................................................................

void
DoxyLexer::init ()
{
	%% write init;
}

void
DoxyLexer::exec ()
{
	%% write exec;
}

//.............................................................................

} // namespace ct
} // namespace jnc