Corrected docs of the xpath lexer.

Corrected docs of the xpath lexer.

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Ragel lexer for lexing XPath expressions.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };
        star   = '*' @{ add_token(:T_STAR) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = [a-zA-Z]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = digit+;
        float   = digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote+ dquote);
        string_squote = (squote ^squote+ squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        operator = '|'
          | 'and'
          | 'or'
          | '+'
          | 'div'
          | 'mod'
          | '='
          | '!='
          | '<'
          | '>'
          | '<='
          | '>=';

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.
        space_operator = space ('*' | '-') space;

        action emit_operator {
          emit(:T_OP, ts, te)
        }

        action emit_space_operator {
          emit(:T_OP, ts + 1, te - 1)
        }

        # Machine that handles the lexing of data inside an XPath predicate.
        # When bumping into a "]" the lexer jumps back to the `main` machine.
        predicate := |*
          whitespace | slash | lparen | rparen | comma | colon | star;

          operator       => emit_operator;
          space_operator => emit_space_operator;

          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;

          ']' => {
            add_token(:T_RBRACK)
            fnext main;
          };
        *|;

        main := |*
          whitespace | slash | lparen | rparen | comma | colon | star;

          '[' => {
            add_token(:T_LBRACK)
            fnext predicate;
          };

          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Ragel lexer for lexing XPath expressions.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };
        star   = '*' @{ add_token(:T_STAR) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = [a-zA-Z]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = digit+;
        float   = digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote+ dquote);
        string_squote = (squote ^squote+ squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        operator = '|'
          | 'and'
          | 'or'
          | '+'
          | 'div'
          | 'mod'
          | '='
          | '!='
          | '<'
          | '>'
          | '<='
          | '>=';

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.
        space_operator = space ('*' | '-') space;

        action emit_operator {
          emit(:T_OP, ts, te)
        }

        action emit_space_operator {
          emit(:T_OP, ts + 1, te - 1)
        }

        # Machine that handles the lexing of data inside an XPath predicate.
        # When bumping into a "]" the lexer jumps back to the `main` machine.
        predicate := |*
          whitespace | slash | lparen | rparen | comma | colon | star;

          operator       => emit_operator;
          space_operator => emit_space_operator;

          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;

          ']' => {
            add_token(:T_RBRACK)
            fnext main;
          };
        *|;

        main := |*
          whitespace | slash | lparen | rparen | comma | colon | star;

          '[' => {
            add_token(:T_LBRACK)
            fnext predicate;
          };

          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga


check for max @id: prevent malicious schemas from chewing up too much memory

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

int calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  size_t j;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // our ragel machine will initialise these variables, assignments here are just to silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    if (!err) err = 1; \
    goto err_bail; \
  } while(0)

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0 || curr_item_index > 1000000)
        PARSE_ERROR("@id way out of range");

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_NESTED && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("nested qstructs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].nested_name = curr_item.nested_name;
      def->items[curr_item_index].nested_name_len = curr_item.nested_name_len;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{ curr_line++; };
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    lc_alpha_u = [a-z] | '_';
    uc_alpha_u = [A-Z] | '_';
    identifier = lc_alpha_u $!{ PARSE_ERROR("qstruct item names must start with lowercase letters"); }
                 alnum_u*;
    identifier_with_package = uc_alpha_u $!{ PARSE_ERROR("qstruct names must start with uppercase letters"); }
                              alnum_u* ('::' uc_alpha_u alnum_u*)*;
    integer = digit+;

    ws = whitespace_char |
         '#' [^\n]* newline |
         '/*' (any_count_line* - (any_count_line* '*/' any_count_line*)) '*/'
      ;

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           identifier_with_package >{ curr_item.nested_name = p; curr_item.type = QSTRUCT_TYPE_NESTED; }
                                   %{ curr_item.nested_name_len = p - curr_item.nested_name; }
      ;

    array_spec = '['
                   ws*
                   integer >{ curr_item.fixed_array_size = 0; }
                           @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                   ws*
                 ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                  |

                 '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
      ;

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec? $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); }
      ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?
      ;

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  // reverse def list
  for (curr_def = NULL; def;) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }
  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    for(j=0; j<def->num_items; j++) {
      if ((def->items[j].type & 0xFFFF) == QSTRUCT_TYPE_NESTED) {
        HASH_FIND(hh, def_hash_by_name, def->items[j].nested_name, def->items[j].nested_name_len, def_lookup);
        if (!def_lookup)
          PARSE_ERROR("type '%.*s' referred to before being defined", (int) def->items[j].nested_name_len, def->items[j].nested_name);

        def->items[j].nested_def = def_lookup;
      }
    }

    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    if (calculate_qstruct_packing(curr_def) < 0)
      PARSE_ERROR("memory error in packing");
  }

  // Success!
  goto cleanup;

  #undef PARSE_ERROR
  err_bail:

  for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {}
  while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++;
  for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {}
  memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start);
  *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0';
  snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf);

  if (def) {
    free_qstruct_definitions(def);
    def = NULL;
  }

  cleanup:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>

#include "qstruct/compiler.h"

int calculate_qstruct_packing(struct qstruct_definition *);

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  size_t j;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def, *curr_def, *temp_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;
  struct qstruct_definition *def_hash_by_name = NULL, *def_lookup;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  struct qstruct_item *item_hash_by_name = NULL, *item_lookup;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // Parsing phase

  // our ragel machine will initialise these variables, assignments here are just to silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    if (!err) err = 1; \
    goto err_bail; \
  } while(0)

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_NESTED && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("nested qstructs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].nested_name = curr_item.nested_name;
      def->items[curr_item_index].nested_name_len = curr_item.nested_name_len;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      assert(!item_hash_by_name);

      for(i=0; i<=largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);

        HASH_FIND(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, item_lookup);
        if (item_lookup)
          PARSE_ERROR("duplicate item name '%.*s'", (int) def->items[i].name_len, def->items[i].name);

        HASH_ADD_KEYPTR(hh, item_hash_by_name, def->items[i].name, def->items[i].name_len, &def->items[i]);
      }

      HASH_CLEAR(hh, item_hash_by_name);

      def[0].num_items = largest_item+1;
    }

    newline = '\n' @{ curr_line++; };
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    lc_alpha_u = [a-z] | '_';
    uc_alpha_u = [A-Z] | '_';
    identifier = lc_alpha_u $!{ PARSE_ERROR("qstruct item names must start with lowercase letters"); }
                 alnum_u*;
    identifier_with_package = uc_alpha_u $!{ PARSE_ERROR("qstruct names must start with uppercase letters"); }
                              alnum_u* ('::' uc_alpha_u alnum_u*)*;
    integer = digit+;

    ws = whitespace_char |
         '#' [^\n]* newline |
         '/*' (any_count_line* - (any_count_line* '*/' any_count_line*)) '*/'
      ;

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           identifier_with_package >{ curr_item.nested_name = p; curr_item.type = QSTRUCT_TYPE_NESTED; }
                                   %{ curr_item.nested_name_len = p - curr_item.nested_name; }
      ;

    array_spec = '['
                   ws*
                   integer >{ curr_item.fixed_array_size = 0; }
                           @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                   ws*
                 ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                  |

                 '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
      ;

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
                      $!{ PARSE_ERROR("invalid identifier"); }
           ws+
           '@' $!{ PARSE_ERROR("expected @ id"); }
           integer >{ curr_item_index = 0; }
                   @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           ws*
           array_spec? $!{ PARSE_ERROR("invalid array specifier"); }
           ws* ';' $!{ PARSE_ERROR("missing semi-colon"); }
      ;

    qstruct = ws*
              ( [qQ] 'struct' ) >init_qstruct
                                $!{ PARSE_ERROR("expected qstruct definition"); }
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct
             (ws* ';')?
      ;

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  // Compilation phase

  // reverse def list
  for (curr_def = NULL; def;) {
    temp_def = def->next;
    def->next = curr_def;
    curr_def = def;
    def = temp_def;
  }
  def = curr_def;

  assert(!def_hash_by_name);

  for(curr_def = def; curr_def; curr_def = curr_def->next) {
    for(j=0; j<def->num_items; j++) {
      if ((def->items[j].type & 0xFFFF) == QSTRUCT_TYPE_NESTED) {
        HASH_FIND(hh, def_hash_by_name, def->items[j].nested_name, def->items[j].nested_name_len, def_lookup);
        if (!def_lookup)
          PARSE_ERROR("type '%.*s' referred to before being defined", (int) def->items[j].nested_name_len, def->items[j].nested_name);

        def->items[j].nested_def = def_lookup;
      }
    }

    HASH_FIND(hh, def_hash_by_name, curr_def->name, curr_def->name_len, def_lookup);
    if (def_lookup)
      PARSE_ERROR("duplicate def name '%.*s'", (int) curr_def->name_len, curr_def->name);

    HASH_ADD_KEYPTR(hh, def_hash_by_name, curr_def->name, curr_def->name_len, curr_def);

    if (calculate_qstruct_packing(curr_def) < 0)
      PARSE_ERROR("memory error in packing");
  }

  // Success!
  goto cleanup;

  #undef PARSE_ERROR
  err_bail:

  for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {}
  while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++;
  for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {}
  memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start);
  *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0';
  snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf);

  if (def) {
    free_qstruct_definitions(def);
    def = NULL;
  }

  cleanup:

  HASH_CLEAR(hh, item_hash_by_name);
  HASH_CLEAR(hh, def_hash_by_name);

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}


Cleaned up lexer rules a bit.

Cleaned up lexer rules a bit.

There's no benefit to adding variables for angle brackets and such, it's much
easier to grok to just use them directly.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # Doctypes are treated with some extra care on lexer level to make the
      # parser's life easier. If they were treated as regular text it would be
      # a pain to specify a proper doctype in Racc since it can't match on a
      # token's value (only on its type).
      #
      # Doctype parsing is also relaxed compared to the W3 specification. For
      # example, the specification defines 4 doctype formats each having
      # different rules. Because Oga doesn't really use the doctype for
      # anything we'll just slap all the formats into a single rule. Easy
      # enough.
      doctype = '<' whitespace* '!' whitespace* 'DOCTYPE'i whitespace*
        'HTML'i whitespace* any* '>';

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_start => {
          t(:T_CDATA_START)

          @cdata_buffer = ''
        };

        cdata_end => {
          add_token(:T_TEXT, @cdata_buffer)
          @cdata_buffer = nil

          t(:T_CDATA_END)

          fgoto main;
        };

        # Consume everything else character by character and store it in a
        # separate buffer.
        any => { @cdata_buffer << text };
      *|;

      main := |*
        whitespace => { t(:T_SPACE) };
        newline    => { t(:T_NEWLINE); advance_line };

        doctype  => { t(:T_DOCTYPE) };

        # Jump to the cdata machine right away without processing anything.
        cdata_start >{ fhold; fgoto cdata; };

        # General rules and actions.
        '<' => { t(:T_SMALLER) };
        '>' => { t(:T_GREATER) };
        '/' => { t(:T_SLASH) };
        '"' => { t(:T_DQUOTE) };
        "'" => { t(:T_SQUOTE) };
        '-' => { t(:T_DASH) };
        ']' => { t(:T_RBRACKET) };
        '[' => { t(:T_LBRACKET) };
        ':' => { t(:T_COLON) };
        '!' => { t(:T_BANG) };
        '=' => { t(:T_EQUALS) };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n';
      whitespace = [ \t];

      any_escaped = /\\./;

      smaller  = '<';
      greater  = '>';
      slash    = '/';
      bang     = '!';
      equals   = '=';
      colon    = ':';
      dash     = '-';
      lbracket = '[';
      rbracket = ']';

      s_quote = "'";
      d_quote = '"';

      # FIXME: there really should be a better way of doing this.
      text = (any - s_quote - d_quote - equals - bang - slash -
        greater - smaller - whitespace - newline - colon - dash -
        lbracket - rbracket)+;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # Doctypes are treated with some extra care on lexer level to make the
      # parser's life easier. If they were treated as regular text it would be
      # a pain to specify a proper doctype in Racc since it can't match on a
      # token's value (only on its type).
      #
      # Doctype parsing is also relaxed compared to the W3 specification. For
      # example, the specification defines 4 doctype formats each having
      # different rules. Because Oga doesn't really use the doctype for
      # anything we'll just slap all the formats into a single rule. Easy
      # enough.
      doctype = smaller whitespace* bang whitespace* 'DOCTYPE'i whitespace*
        'HTML'i whitespace* any* greater;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = smaller bang lbracket 'CDATA' lbracket;
      cdata_end   = rbracket rbracket greater;

      cdata := |*
        cdata_start => {
          t(:T_CDATA_START)

          @cdata_buffer = ''
        };

        cdata_end => {
          add_token(:T_TEXT, @cdata_buffer)
          @cdata_buffer = nil

          t(:T_CDATA_END)

          fgoto main;
        };

        # Consume everything else character by character and store it in a
        # separate buffer.
        any => { @cdata_buffer << text };
      *|;

      main := |*
        whitespace => { t(:T_SPACE) };
        newline    => { t(:T_NEWLINE); advance_line };

        doctype  => { t(:T_DOCTYPE) };

        # Jump to the cdata machine right away without processing anything.
        cdata_start >{ fhold; fgoto cdata; };

        # General rules and actions.
        smaller  => { t(:T_SMALLER) };
        greater  => { t(:T_GREATER) };
        slash    => { t(:T_SLASH) };
        d_quote  => { t(:T_DQUOTE) };
        s_quote  => { t(:T_SQUOTE) };
        dash     => { t(:T_DASH) };
        rbracket => { t(:T_RBRACKET) };
        lbracket => { t(:T_LBRACKET) };
        colon    => { t(:T_COLON) };
        bang     => { t(:T_BANG) };
        equals   => { t(:T_EQUALS) };
        text     => { t(:T_TEXT) };
      *|;
    }%%
  end # Lexer
end # Oga


allow quotes only with literal-match-patterns

%%{
  machine hext;

  action error {
    throw_error();
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  cap_var       = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex   = ( [^/][^/]** );
  match_regex = ( [^/][^/]** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          '@' >{ LX_TK_START(TK_BUILTIN); }
              %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '='
          (
            (
              (
                '{' >{ LX_TK_START(TK_CAP_BEGIN); }
                    %{ LX_TK_STOP; }
              )
              (
                cap_var >{ LX_TK_START(TK_CAP_VAR); }
                        %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START(TK_MATCH_REGEX); }
                              %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                '"'
                (
                  match_literal >{ LX_TK_START(TK_MATCH_LITERAL); }
                                %{ LX_TK_STOP; }
                )
                '"'
              )
            )
          )
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          nth_child >{ LX_TK_START(TK_NTH_CHILD); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )?
      ):>
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    throw_error();
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  cap_var       = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex   = ( [^/][^/]** );
  match_regex = ( [^/][^/]** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          '@' >{ LX_TK_START(TK_BUILTIN); }
              %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              (
                '{' >{ LX_TK_START(TK_CAP_BEGIN); }
                    %{ LX_TK_STOP; }
              )
              (
                cap_var >{ LX_TK_START(TK_CAP_VAR); }
                        %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START(TK_MATCH_REGEX); }
                              %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                match_literal >{ LX_TK_START(TK_MATCH_LITERAL); }
                              %{ LX_TK_STOP; }
              )
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          nth_child >{ LX_TK_START(TK_NTH_CHILD); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )?
      ):>
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%


Fix typo in code comment

Fix typo in code comment

See:

  https://wincent.com/issues/1754

Signed-off-by: Wincent Colaiuta <c76ac2e9cd86441713c7dfae92c451f3e6d17fdc@wincent.com>

// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include "wikitext.h"
#include <stdio.h>

#define EMIT(t)     do { out->type = t; out->stop = p + 1; out->column_stop += (out->stop - out->start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)
#define AT_END()    (p + 1 == pe)
#define DISTANCE()  (p + 1 - ts)
#define NEXT_CHAR() (*(p + 1))

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        out->code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    # simple approximation for matching email addresses; not quite RFC 2822!
    user                = (alnum | [_\.] | '-')+ ;
    tld                 = alpha{2,5} ;
    domain              = (alnum+ '.')+ tld ;
    mail                = user '@' domain ;

    uri_chars           = (alnum | [@$&'(\*\+=%_~/#] | '-')+ ;
    special_uri_chars   = ([:!\(\),;\.\?])+ ;
    uri                 = ('mailto:'i mail) |
                          (('http'i [sS]? '://' | 'ftp://'i | 'svn://'i) uri_chars (special_uri_chars uri_chars)*) ;
    path                = '/' ([a-zA-Z0-9_\-.]+ '/'?)* ;

    main := |*

        '<nowiki>'i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        '</nowiki>'i
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        '
'i
        {
            EMIT(PRE_START);
            fbreak;
        };

        '
'
        {
            EMIT(PRE_START);
            fbreak;
        };

        '
'i
        {
            EMIT(PRE_END);
            fbreak;
        };

        '
'i
        {
            EMIT(BLOCKQUOTE_START);
            fbreak;
        };

        '
'i
        {
            EMIT(BLOCKQUOTE_END);
            fbreak;
        };

        "'"{1,5}
        {
            if (DISTANCE() == 5)
                EMIT(STRONG_EM);
            else if (DISTANCE() == 4)
            {
                p--;
                EMIT(STRONG_EM);
            }
            else if (DISTANCE() == 3)
                EMIT(STRONG);
            else if (DISTANCE() == 2)
                EMIT(EM);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_START);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_END);
            fbreak;
        };

        ''i
        {
            EMIT(EM_START);
            fbreak;
        };

        ''i
        {
            EMIT(EM_END);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        '<tt>'i
        {
            EMIT(TT_START);
            fbreak;
        };

        '</tt>'i
        {
            EMIT(TT_END);
            fbreak;
        };

        # shorthand for
 and

        '>' @mark ' '?
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        ' ' @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(PRE);
            }
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (out->column_start == 1              ||
                last_token_type == OL               ||
                last_token_type == UL               ||
                last_token_type == BLOCKQUOTE       ||
                last_token_type == BLOCKQUOTE_START)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (out->column_start == 1              ||
                last_token_type == OL               ||
                last_token_type == UL               ||
                last_token_type == BLOCKQUOTE       ||
                last_token_type == BLOCKQUOTE_START)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='+ @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE || last_token_type == BLOCKQUOTE_START)
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_START);
                else if (DISTANCE() == 2)
                    EMIT(H2_START);
                else if (DISTANCE() == 3)
                    EMIT(H3_START);
                else if (DISTANCE() == 4)
                    EMIT(H4_START);
                else if (DISTANCE() == 5)
                    EMIT(H5_START);
                else if (DISTANCE() == 6)
                    EMIT(H6_START);
                else if (DISTANCE() > 6)
                {
                    p = ts + 6;
                    EMIT(H6_START);
                }
            }
            else if (AT_END() || NEXT_CHAR() == '\n' || NEXT_CHAR() == '\r')
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_END);
                else if (DISTANCE() == 2)
                    EMIT(H2_END);
                else if (DISTANCE() == 3)
                    EMIT(H3_END);
                else if (DISTANCE() == 4)
                    EMIT(H4_END);
                else if (DISTANCE() == 5)
                    EMIT(H5_END);
                else if (DISTANCE() == 6)
                    EMIT(H6_END);
                else if (DISTANCE() > 6)
                {
                    p -= 6; // will scan the H6 on the next scan
                    EMIT(PRINTABLE);
                }
            }
            else
            {
                // note that a H*_END token will never match before a BLOCKQUOTE_END
                REWIND();
                EMIT(PRINTABLE);
            }
            fbreak;
        };

        uri
        {
            EMIT(URI);
            fbreak;
        };

        mail
        {
            EMIT(MAIL);
            fbreak;
        };

        path
        {
            EMIT(PATH);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        '{{'
        {
            EMIT(IMG_START);
            fbreak;
        };

        '}}'
        {
            EMIT(IMG_END);
            fbreak;
        };

        '{'
        {
            EMIT(LEFT_CURLY);
            fbreak;
        };

        '}'
        {
            EMIT(RIGHT_CURLY);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            out->column_stop = 1;
            out->line_stop++;
            fbreak;
        };

        # must tokenize these separately from the other PRINTABLE characters otherwise a string like:
        #   See http://example.com/.
        # will get greedily tokenized as PRINTABLE, SPACE, PRINTABLE rather than PRINTABLE, SPACE, URI, SPECIAL_URI_CHARS
        # this also applies to MAIL tokenization and input strings like:
        #   Email me (user@example.com) for more info.
        special_uri_chars
        {
            EMIT(SPECIAL_URI_CHARS);
            fbreak;
        };

        alnum+
        {
            EMIT(ALNUM);
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # we skip space (0x20), exclamation mark (0x21), quote (0x22), hash (0x23), ampersand (0x26), apostrophe (0x27),
        # left parenthesis (0x28), right parenthesis (0x29), numbers (0x30..0x39), asterisk (0x2a), comma (0x2c), period (0x2e),
        # colon (0x3a), semi-colon (0x3b), less than (0x3c), equals (0x3d), greater than (0x3e), question mark (0x3f), uppercase
        # letters (0x41..0x5a), left bracket (0x5b), right bracket (0x5d), backtick (0x60), lowercase letters (0x61..0x7a), left
        # curly brace (0x7b), vertical bar (0x7c) and right curly brace (0x7d).
        (0x24..0x25 | 0x2b | 0x2d | 0x2f | 0x40 | 0x5c | 0x5e..0x5f | 0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             @non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         @two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              @three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   @four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            out->column_stop = out->column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
void next_token(token_t *out, token_t *last_token, char *p, char *pe)
{
    int last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        out->line_start     = out->line_stop    = last_token->line_stop;
        out->column_start   = out->column_stop  = last_token->column_stop;
    }
    else
    {
        out->line_start     = 1;
        out->column_start   = 1;
        out->line_stop      = 1;
        out->column_stop    = 1;
    }
    out->type       = NO_TOKEN;
    out->code_point = 0;
    out->start      = p;
    if (p == pe)
    {
        // all done, have reached end of input
        out->stop  = p;
        out->type  = END_OF_FILE;
        return;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextParserError, "failed before finding a token");
    else if (out->type == NO_TOKEN)
        rb_raise(eWikitextParserError, "failed to produce a token");
}

// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include "wikitext.h"
#include <stdio.h>

#define EMIT(t)     do { out->type = t; out->stop = p + 1; out->column_stop += (out->stop - out->start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)
#define AT_END()    (p + 1 == pe)
#define DISTANCE()  (p + 1 - ts)
#define NEXT_CHAR() (*(p + 1))

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        out->code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    # simple approximation for matching email addresses; not quite RFC 2822!
    user                = (alnum | [_\.] | '-')+ ;
    tld                 = alpha{2,5} ;
    domain              = (alnum+ '.')+ tld ;
    mail                = user '@' domain ;

    uri_chars           = (alnum | [@$&'(\*\+=%_~/#] | '-')+ ;
    special_uri_chars   = ([:!\(\),;\.\?])+ ;
    uri                 = ('mailto:'i mail) |
                          (('http'i [sS]? '://' | 'ftp://'i | 'svn://'i) uri_chars (special_uri_chars uri_chars)*) ;
    path                = '/' ([a-zA-Z0-9_\-.]+ '/'?)* ;

    main := |*

        '<nowiki>'i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        '</nowiki>'i
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        '
'i
        {
            EMIT(PRE_START);
            fbreak;
        };

        '
'
        {
            EMIT(PRE_START);
            fbreak;
        };

        '
'i
        {
            EMIT(PRE_END);
            fbreak;
        };

        '
'i
        {
            EMIT(BLOCKQUOTE_START);
            fbreak;
        };

        '
'i
        {
            EMIT(BLOCKQUOTE_END);
            fbreak;
        };

        "'"{1,5}
        {
            if (DISTANCE() == 5)
                EMIT(STRONG_EM);
            else if (DISTANCE() == 4)
            {
                p--;
                EMIT(STRONG_EM);
            }
            else if (DISTANCE() == 3)
                EMIT(STRONG);
            else if (DISTANCE() == 2)
                EMIT(EM);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_START);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_END);
            fbreak;
        };

        ''i
        {
            EMIT(EM_START);
            fbreak;
        };

        ''i
        {
            EMIT(EM_END);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        '<tt>'i
        {
            EMIT(TT_START);
            fbreak;
        };

        '</tt>'i
        {
            EMIT(TT_END);
            fbreak;
        };

        # shorthand for
 and

        '>' @mark ' '?
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        ' ' @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(PRE);
            }
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (out->column_start == 1              ||
                last_token_type == OL               ||
                last_token_type == UL               ||
                last_token_type == BLOCKQUOTE       ||
                last_token_type == BLOCKQUOTE_START)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (out->column_start == 1              ||
                last_token_type == OL               ||
                last_token_type == UL               ||
                last_token_type == BLOCKQUOTE       ||
                last_token_type == BLOCKQUOTE_START)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='+ @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE || last_token_type == BLOCKQUOTE_START)
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_START);
                else if (DISTANCE() == 2)
                    EMIT(H2_START);
                else if (DISTANCE() == 3)
                    EMIT(H3_START);
                else if (DISTANCE() == 4)
                    EMIT(H4_START);
                else if (DISTANCE() == 5)
                    EMIT(H5_START);
                else if (DISTANCE() == 6)
                    EMIT(H6_START);
                else if (DISTANCE() > 6)
                {
                    p = ts + 6;
                    EMIT(H6_START);
                }
            }
            else if (AT_END() || NEXT_CHAR() == '\n' || NEXT_CHAR() == '\r')
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_END);
                else if (DISTANCE() == 2)
                    EMIT(H2_END);
                else if (DISTANCE() == 3)
                    EMIT(H3_END);
                else if (DISTANCE() == 4)
                    EMIT(H4_END);
                else if (DISTANCE() == 5)
                    EMIT(H5_END);
                else if (DISTANCE() == 6)
                    EMIT(H6_END);
                else if (DISTANCE() > 6)
                {
                    p -= 6; // will scan the H6 on the next scan
                    EMIT(PRINTABLE);
                }
            }
            else
            {
                // note that a H*_END token will never match before a BLOCKQUOTE_END
                REWIND();
                EMIT(PRINTABLE);
            }
            fbreak;
        };

        uri
        {
            EMIT(URI);
            fbreak;
        };

        mail
        {
            EMIT(MAIL);
            fbreak;
        };

        path
        {
            EMIT(PATH);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        '{{'
        {
            EMIT(IMG_START);
            fbreak;
        };

        '}}'
        {
            EMIT(IMG_END);
            fbreak;
        };

        '{'
        {
            EMIT(LEFT_CURLY);
            fbreak;
        };

        '}'
        {
            EMIT(RIGHT_CURLY);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            out->column_stop = 1;
            out->line_stop++;
            fbreak;
        };

        # must tokenize these separately from the other PRINTABLE characters otherwise a string like:
        #   See http://example.com/.
        # will get greedily tokenized as PRINABLE, SPACE, PRINTABLE rather than PRINTABLE, SPACE, URI, SPECIAL_URI_CHARS
        # this also applies to MAIL tokenization and input strings like:
        #   Email me (user@example.com) for more info.
        special_uri_chars
        {
            EMIT(SPECIAL_URI_CHARS);
            fbreak;
        };

        alnum+
        {
            EMIT(ALNUM);
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # we skip space (0x20), exclamation mark (0x21), quote (0x22), hash (0x23), ampersand (0x26), apostrophe (0x27),
        # left parenthesis (0x28), right parenthesis (0x29), numbers (0x30..0x39), asterisk (0x2a), comma (0x2c), period (0x2e),
        # colon (0x3a), semi-colon (0x3b), less than (0x3c), equals (0x3d), greater than (0x3e), question mark (0x3f), uppercase
        # letters (0x41..0x5a), left bracket (0x5b), right bracket (0x5d), backtick (0x60), lowercase letters (0x61..0x7a), left
        # curly brace (0x7b), vertical bar (0x7c) and right curly brace (0x7d).
        (0x24..0x25 | 0x2b | 0x2d | 0x2f | 0x40 | 0x5c | 0x5e..0x5f | 0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             @non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         @two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              @three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   @four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            out->column_stop = out->column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
void next_token(token_t *out, token_t *last_token, char *p, char *pe)
{
    int last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        out->line_start     = out->line_stop    = last_token->line_stop;
        out->column_start   = out->column_stop  = last_token->column_stop;
    }
    else
    {
        out->line_start     = 1;
        out->column_start   = 1;
        out->line_stop      = 1;
        out->column_stop    = 1;
    }
    out->type       = NO_TOKEN;
    out->code_point = 0;
    out->start      = p;
    if (p == pe)
    {
        // all done, have reached end of input
        out->stop  = p;
        out->type  = END_OF_FILE;
        return;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextParserError, "failed before finding a token");
    else if (out->type == NO_TOKEN)
        rb_raise(eWikitextParserError, "failed to produce a token");
}


expand allowable incomplete set nodes

expand allowable incomplete set nodes

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';
  include 'list.rl';

  action add_name {
    $stderr.puts "SET add_name"
    key = @buffers.delete(:ident)
    @buffers[:set_name] = key
  }

  action add_ident_value {
    $stderr.puts "SET add_ident_value"
    ident = @buffers.delete(:ident)
    @buffers[:set_value] = ident
  }

  action add_string_value {
    $stderr.puts "SET add_string_value"
    string = @buffers.delete(:string)
    @buffers[:set_value] = string
  }

  action add_list_value {
    $stderr.puts "SET add_list_value"
    list = @buffers.delete(:list)
    @buffers[:set_value] = list
  }

  action set_end {
    $stderr.puts "SET set_end"
    set_node = set()
    completed = true

    name = @buffers.delete(:set_name)
    if name.nil?
      completed = false
    else
      set_node <<= name
    end

    value = @buffers.delete(:set_value)
    if value.nil?
      completed = false
    else
      set_node <<= value
    end
    set_node.complete = completed
    @buffers[:set] = set_node
  }

  action yield_set {
    $stderr.puts "SET yield_set"
    yield @buffers[:set]
  }

  action set_node_eof {
    $stderr.puts "SET set_node_eof"
    name = @buffers.delete(:set_name)
    set_node = set(name)
    completed = name.complete
    if @buffers.key?(:string)
      value = @buffers.delete(:string)
      set_node <<= value
      completed = completed && value.complete
    elsif @buffers.key?(:ident)
      value = @buffers.delete(:ident)
      set_node <<= value
      completed = completed && value.complete
    elsif @buffers.key?(:list)
      value = @buffers.delete(:list)
      set_node <<= value
      completed = completed && value.complete
    end
    set_node.complete = completed
    yield set_node
  }

  key =
    an_ident
    %add_name
    ;

  ident_value =
    an_ident
    %add_ident_value
    ;

  string_value =
    a_string
    %add_string_value
    ;

  list_value =
    a_list
    %add_list_value
    ;

  value =
    ident_value |
    string_value |
    list_value
    ;

  set_node :=
    KW_SET
    SP+
    an_ident
    %add_name
    @eof(set_node_eof)
    SP+
    EQL?
    SP+?
    value?
    @eof(set_node_eof)
    %set_end
    %yield_set
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module BELScript
      module Set

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @accum_set  = false
            @have_set   = false
            @have_name  = false
            @have_value = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::BELScript::Set.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';
  include 'list.rl';

  action add_key {
    $stderr.puts "SET add_key"
    key = @buffers.delete(:ident)
    @buffers[:set_name] = key
  }

  action add_ident_value {
    $stderr.puts "SET add_ident_value"
    ident = @buffers.delete(:ident)
    @buffers[:set_value] = ident
  }

  action add_string_value {
    $stderr.puts "SET add_string_value"
    string = @buffers.delete(:string)
    @buffers[:set_value] = string
  }

  action add_list_value {
    $stderr.puts "SET add_list_value"
    list = @buffers.delete(:list)
    @buffers[:set_value] = list
  }

  action set_end {
    $stderr.puts "SET set_end"
    name = @buffers.delete(:set_name)
    value = @buffers.delete(:set_value)
    set_node = set(name, value, complete: name.complete && value.complete)
    @buffers[:set] = set_node
  }

  action yield_set {
    $stderr.puts "SET yield_set"
    yield @buffers[:set]
  }

  action set_node_eof {
    $stderr.puts "SET set_node_eof"
    name = @buffers.delete(:set_name)
    set_node = set(name)
    completed = name.complete
    if @buffers.key?(:string)
      value = @buffers.delete(:string)
      set_node <<= value
      completed = completed && value.complete
    elsif @buffers.key?(:ident)
      value = @buffers.delete(:ident)
      set_node <<= value
      completed = completed && value.complete
    elsif @buffers.key?(:list)
      value = @buffers.delete(:list)
      set_node <<= value
      completed = completed && value.complete
    end
    set_node.complete = completed
    yield set_node
  }

  SET_KW =
    [sS]
    [eE]
    [tT]
    ;

  key =
    an_ident
    %add_key
    ;

  ident_value =
    an_ident
    %add_ident_value
    ;

  string_value =
    a_string
    %add_string_value
    ;

  list_value =
    a_list
    %add_list_value
    ;

  value =
    ident_value |
    string_value |
    list_value
    ;

  set_node :=
    SET_KW
    SP+
    an_ident
    %add_key
    @eof(set_node_eof)
    SP+
    EQL
    SP+
    value
    @eof(set_node_eof)
    %set_end
    %yield_set
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module BELScript
      module Set

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            @accum_set  = false
            @have_set   = false
            @have_name  = false
            @have_value = false
            data        = @content.unpack('C*')
            p           = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::BELScript::Set.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Ragel: some text

Ragel: some text

# Ragel lexer for VCF-header
#
# This is a partial lexer for the VCF file format. Bio-vcf uses that
# to generate meta information in (for example) JSON format. The
# advantage of using a full state engine is that it allows for easy
# parsing of key-value pairs with syntax checking and, for example,
# escaped quotes in quoted string values. Currently we validate ID and
# Number fields only. If should not be hard to continue adding syntax
# checking features.

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end