Add EXTEND token to lexer

Add EXTEND token to lexer

%%{
  machine graphql_lexer;

  IDENTIFIER =    [_A-Za-z][_0-9A-Za-z]*;
  NEWLINE =       [\c\r\n];
  BLANK   =       [, \t]+;
  COMMENT =       '#' [^\n\r]*;
  INT =           '-'? ('0'|[1-9][0-9]*);
  FLOAT_DECIMAL = '.'[0-9]+;
  FLOAT_EXP =     ('e' | 'E')?('+' | '-')?[0-9]+;
  FLOAT =         INT FLOAT_DECIMAL? FLOAT_EXP?;
  ON =            'on';
  FRAGMENT =      'fragment';
  TRUE =          'true';
  FALSE =         'false';
  NULL =          'null';
  QUERY =         'query';
  MUTATION =      'mutation';
  SUBSCRIPTION =  'subscription';
  SCHEMA =        'schema';
  SCALAR =        'scalar';
  TYPE =          'type';
  EXTEND =        'extend';
  IMPLEMENTS =    'implements';
  INTERFACE =     'interface';
  UNION =         'union';
  ENUM =          'enum';
  INPUT =         'input';
  DIRECTIVE =     'directive';
  LCURLY =        '{';
  RCURLY =        '}';
  LPAREN =        '(';
  RPAREN =        ')';
  LBRACKET =      '[';
  RBRACKET =      ']';
  COLON =         ':';
  QUOTE =         '"';
  BLOCK_QUOTE =   '"""';
  ESCAPED_BLOCK_QUOTE = '\\"""';
  BLOCK_STRING_CHAR = (ESCAPED_BLOCK_QUOTE | ^'"""');
  ESCAPED_QUOTE = '\\"';
  STRING_CHAR =   (ESCAPED_QUOTE | ^'"');
  VAR_SIGN =      '$';
  DIR_SIGN =      '@';
  ELLIPSIS =      '...';
  EQUALS =        '=';
  BANG =          '!';
  PIPE =          '|';
  AMP =           '&';

  QUOTED_STRING = QUOTE STRING_CHAR* QUOTE;
  BLOCK_STRING = BLOCK_QUOTE BLOCK_STRING_CHAR* BLOCK_QUOTE;
  # catch-all for anything else. must be at the bottom for precedence.
  UNKNOWN_CHAR =         /./;

  main := |*
    INT           => { emit(:INT, ts, te, meta) };
    FLOAT         => { emit(:FLOAT, ts, te, meta) };
    ON            => { emit(:ON, ts, te, meta) };
    FRAGMENT      => { emit(:FRAGMENT, ts, te, meta) };
    TRUE          => { emit(:TRUE, ts, te, meta) };
    FALSE         => { emit(:FALSE, ts, te, meta) };
    NULL          => { emit(:NULL, ts, te, meta) };
    QUERY         => { emit(:QUERY, ts, te, meta) };
    MUTATION      => { emit(:MUTATION, ts, te, meta) };
    SUBSCRIPTION  => { emit(:SUBSCRIPTION, ts, te, meta) };
    SCHEMA        => { emit(:SCHEMA, ts, te, meta) };
    SCALAR        => { emit(:SCALAR, ts, te, meta) };
    TYPE          => { emit(:TYPE, ts, te, meta) };
    EXTEND        => { emit(:EXTEND, ts, te, meta) };
    IMPLEMENTS    => { emit(:IMPLEMENTS, ts, te, meta) };
    INTERFACE     => { emit(:INTERFACE, ts, te, meta) };
    UNION         => { emit(:UNION, ts, te, meta) };
    ENUM          => { emit(:ENUM, ts, te, meta) };
    INPUT         => { emit(:INPUT, ts, te, meta) };
    DIRECTIVE     => { emit(:DIRECTIVE, ts, te, meta) };
    RCURLY        => { emit(:RCURLY, ts, te, meta) };
    LCURLY        => { emit(:LCURLY, ts, te, meta) };
    RPAREN        => { emit(:RPAREN, ts, te, meta) };
    LPAREN        => { emit(:LPAREN, ts, te, meta) };
    RBRACKET      => { emit(:RBRACKET, ts, te, meta) };
    LBRACKET      => { emit(:LBRACKET, ts, te, meta) };
    COLON         => { emit(:COLON, ts, te, meta) };
    QUOTED_STRING => { emit_string(ts, te, meta, block: false) };
    BLOCK_STRING  => { emit_string(ts, te, meta, block: true) };
    VAR_SIGN      => { emit(:VAR_SIGN, ts, te, meta) };
    DIR_SIGN      => { emit(:DIR_SIGN, ts, te, meta) };
    ELLIPSIS      => { emit(:ELLIPSIS, ts, te, meta) };
    EQUALS        => { emit(:EQUALS, ts, te, meta) };
    BANG          => { emit(:BANG, ts, te, meta) };
    PIPE          => { emit(:PIPE, ts, te, meta) };
    AMP           => { emit(:AMP, ts, te, meta) };
    IDENTIFIER    => { emit(:IDENTIFIER, ts, te, meta) };
    COMMENT       => { record_comment(ts, te, meta) };

    NEWLINE => {
      meta[:line] += 1
      meta[:col] = 1
    };

    BLANK   => { meta[:col] += te - ts };

    UNKNOWN_CHAR => { emit(:UNKNOWN_CHAR, ts, te, meta) };

  *|;
}%%

module GraphQL
  module Language
    module Lexer
      def self.tokenize(query_string)
        run_lexer(query_string)
      end

      # Replace any escaped unicode or whitespace with the _actual_ characters
      # To avoid allocating more strings, this modifies the string passed into it
      def self.replace_escaped_characters_in_place(raw_string)
        raw_string.gsub!(ESCAPES, ESCAPES_REPLACE)
        raw_string.gsub!(UTF_8, &UTF_8_REPLACE)
        nil
      end

      private

      %% write data;

      def self.run_lexer(query_string)
        data = query_string.unpack("c*")
        eof = data.length

        # Since `Lexer` is a module, store all lexer state
        # in this local variable:
        meta = {
          line: 1,
          col: 1,
          data: data,
          tokens: [],
          previous_token: nil,
        }

        p ||= 0
        pe ||= data.length

        %% write init;

        %% write exec;

        meta[:tokens]
      end

      def self.record_comment(ts, te, meta)
        token = GraphQL::Language::Token.new(
          name: :COMMENT,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )

        meta[:previous_token] = token

        meta[:col] += te - ts
      end

      def self.emit(token_name, ts, te, meta)
        meta[:tokens] << token = GraphQL::Language::Token.new(
          name: token_name,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )
        meta[:previous_token] = token
        # Bump the column counter for the next token
        meta[:col] += te - ts
      end

      ESCAPES = /\\["\\\/bfnrt]/
      ESCAPES_REPLACE = {
        '\\"' => '"',
        "\\\\" => "\\",
        "\\/" => '/',
        "\\b" => "\b",
        "\\f" => "\f",
        "\\n" => "\n",
        "\\r" => "\r",
        "\\t" => "\t",
      }

      UTF_8 = /\\u[\dAa-f]{4}/i
      UTF_8_REPLACE = ->(m) { [m[-4..-1].to_i(16)].pack('U'.freeze) }

      VALID_STRING = /\A(?:[^\\]|#{ESCAPES}|#{UTF_8})*\z/o

      PACK_DIRECTIVE = "c*"
      UTF_8_ENCODING = "UTF-8"

      def self.emit_string(ts, te, meta, block:)
        quotes_length = block ? 3 : 1
        ts += quotes_length
        value = meta[:data][ts...te - quotes_length].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING)
        if block
          value = GraphQL::Language::BlockString.trim_whitespace(value)
        end
        if value !~ VALID_STRING
          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :BAD_UNICODE_ESCAPE,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        else
          replace_escaped_characters_in_place(value)

          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :STRING,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        end

        meta[:previous_token] = token
        meta[:col] += te - ts
      end
    end
  end
end

%%{
  machine graphql_lexer;

  IDENTIFIER =    [_A-Za-z][_0-9A-Za-z]*;
  NEWLINE =       [\c\r\n];
  BLANK   =       [, \t]+;
  COMMENT =       '#' [^\n\r]*;
  INT =           '-'? ('0'|[1-9][0-9]*);
  FLOAT_DECIMAL = '.'[0-9]+;
  FLOAT_EXP =     ('e' | 'E')?('+' | '-')?[0-9]+;
  FLOAT =         INT FLOAT_DECIMAL? FLOAT_EXP?;
  ON =            'on';
  FRAGMENT =      'fragment';
  TRUE =          'true';
  FALSE =         'false';
  NULL =          'null';
  QUERY =         'query';
  MUTATION =      'mutation';
  SUBSCRIPTION =  'subscription';
  SCHEMA =        'schema';
  SCALAR =        'scalar';
  TYPE =          'type';
  IMPLEMENTS =    'implements';
  INTERFACE =     'interface';
  UNION =         'union';
  ENUM =          'enum';
  INPUT =         'input';
  DIRECTIVE =     'directive';
  LCURLY =        '{';
  RCURLY =        '}';
  LPAREN =        '(';
  RPAREN =        ')';
  LBRACKET =      '[';
  RBRACKET =      ']';
  COLON =         ':';
  QUOTE =         '"';
  BLOCK_QUOTE =   '"""';
  ESCAPED_BLOCK_QUOTE = '\\"""';
  BLOCK_STRING_CHAR = (ESCAPED_BLOCK_QUOTE | ^'"""');
  ESCAPED_QUOTE = '\\"';
  STRING_CHAR =   (ESCAPED_QUOTE | ^'"');
  VAR_SIGN =      '$';
  DIR_SIGN =      '@';
  ELLIPSIS =      '...';
  EQUALS =        '=';
  BANG =          '!';
  PIPE =          '|';
  AMP =           '&';

  QUOTED_STRING = QUOTE STRING_CHAR* QUOTE;
  BLOCK_STRING = BLOCK_QUOTE BLOCK_STRING_CHAR* BLOCK_QUOTE;
  # catch-all for anything else. must be at the bottom for precedence.
  UNKNOWN_CHAR =         /./;

  main := |*
    INT           => { emit(:INT, ts, te, meta) };
    FLOAT         => { emit(:FLOAT, ts, te, meta) };
    ON            => { emit(:ON, ts, te, meta) };
    FRAGMENT      => { emit(:FRAGMENT, ts, te, meta) };
    TRUE          => { emit(:TRUE, ts, te, meta) };
    FALSE         => { emit(:FALSE, ts, te, meta) };
    NULL          => { emit(:NULL, ts, te, meta) };
    QUERY         => { emit(:QUERY, ts, te, meta) };
    MUTATION      => { emit(:MUTATION, ts, te, meta) };
    SUBSCRIPTION  => { emit(:SUBSCRIPTION, ts, te, meta) };
    SCHEMA        => { emit(:SCHEMA, ts, te, meta) };
    SCALAR        => { emit(:SCALAR, ts, te, meta) };
    TYPE          => { emit(:TYPE, ts, te, meta) };
    IMPLEMENTS    => { emit(:IMPLEMENTS, ts, te, meta) };
    INTERFACE     => { emit(:INTERFACE, ts, te, meta) };
    UNION         => { emit(:UNION, ts, te, meta) };
    ENUM          => { emit(:ENUM, ts, te, meta) };
    INPUT         => { emit(:INPUT, ts, te, meta) };
    DIRECTIVE     => { emit(:DIRECTIVE, ts, te, meta) };
    RCURLY        => { emit(:RCURLY, ts, te, meta) };
    LCURLY        => { emit(:LCURLY, ts, te, meta) };
    RPAREN        => { emit(:RPAREN, ts, te, meta) };
    LPAREN        => { emit(:LPAREN, ts, te, meta) };
    RBRACKET      => { emit(:RBRACKET, ts, te, meta) };
    LBRACKET      => { emit(:LBRACKET, ts, te, meta) };
    COLON         => { emit(:COLON, ts, te, meta) };
    QUOTED_STRING => { emit_string(ts, te, meta, block: false) };
    BLOCK_STRING  => { emit_string(ts, te, meta, block: true) };
    VAR_SIGN      => { emit(:VAR_SIGN, ts, te, meta) };
    DIR_SIGN      => { emit(:DIR_SIGN, ts, te, meta) };
    ELLIPSIS      => { emit(:ELLIPSIS, ts, te, meta) };
    EQUALS        => { emit(:EQUALS, ts, te, meta) };
    BANG          => { emit(:BANG, ts, te, meta) };
    PIPE          => { emit(:PIPE, ts, te, meta) };
    AMP           => { emit(:AMP, ts, te, meta) };
    IDENTIFIER    => { emit(:IDENTIFIER, ts, te, meta) };
    COMMENT       => { record_comment(ts, te, meta) };

    NEWLINE => {
      meta[:line] += 1
      meta[:col] = 1
    };

    BLANK   => { meta[:col] += te - ts };

    UNKNOWN_CHAR => { emit(:UNKNOWN_CHAR, ts, te, meta) };

  *|;
}%%

module GraphQL
  module Language
    module Lexer
      def self.tokenize(query_string)
        run_lexer(query_string)
      end

      # Replace any escaped unicode or whitespace with the _actual_ characters
      # To avoid allocating more strings, this modifies the string passed into it
      def self.replace_escaped_characters_in_place(raw_string)
        raw_string.gsub!(ESCAPES, ESCAPES_REPLACE)
        raw_string.gsub!(UTF_8, &UTF_8_REPLACE)
        nil
      end

      private

      %% write data;

      def self.run_lexer(query_string)
        data = query_string.unpack("c*")
        eof = data.length

        # Since `Lexer` is a module, store all lexer state
        # in this local variable:
        meta = {
          line: 1,
          col: 1,
          data: data,
          tokens: [],
          previous_token: nil,
        }

        p ||= 0
        pe ||= data.length

        %% write init;

        %% write exec;

        meta[:tokens]
      end

      def self.record_comment(ts, te, meta)
        token = GraphQL::Language::Token.new(
          name: :COMMENT,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )

        meta[:previous_token] = token

        meta[:col] += te - ts
      end

      def self.emit(token_name, ts, te, meta)
        meta[:tokens] << token = GraphQL::Language::Token.new(
          name: token_name,
          value: meta[:data][ts...te].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING),
          line: meta[:line],
          col: meta[:col],
          prev_token: meta[:previous_token],
        )
        meta[:previous_token] = token
        # Bump the column counter for the next token
        meta[:col] += te - ts
      end

      ESCAPES = /\\["\\\/bfnrt]/
      ESCAPES_REPLACE = {
        '\\"' => '"',
        "\\\\" => "\\",
        "\\/" => '/',
        "\\b" => "\b",
        "\\f" => "\f",
        "\\n" => "\n",
        "\\r" => "\r",
        "\\t" => "\t",
      }

      UTF_8 = /\\u[\dAa-f]{4}/i
      UTF_8_REPLACE = ->(m) { [m[-4..-1].to_i(16)].pack('U'.freeze) }

      VALID_STRING = /\A(?:[^\\]|#{ESCAPES}|#{UTF_8})*\z/o

      PACK_DIRECTIVE = "c*"
      UTF_8_ENCODING = "UTF-8"

      def self.emit_string(ts, te, meta, block:)
        quotes_length = block ? 3 : 1
        ts += quotes_length
        value = meta[:data][ts...te - quotes_length].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING)
        if block
          value = GraphQL::Language::BlockString.trim_whitespace(value)
        end
        if value !~ VALID_STRING
          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :BAD_UNICODE_ESCAPE,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        else
          replace_escaped_characters_in_place(value)

          meta[:tokens] << token = GraphQL::Language::Token.new(
            name: :STRING,
            value: value,
            line: meta[:line],
            col: meta[:col],
            prev_token: meta[:previous_token],
          )
        end

        meta[:previous_token] = token
        meta[:col] += te - ts
      end
    end
  end
end


A couple optimizations from benchmarking the Float directives.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline static void swapf(std::string& str, float value) {
      uint32_t x = swap32(*(uint32_t*)(&value));

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline static void swapd(std::string& str, double value) {
      uint64_t x = swap64(*(uint64_t*)(&value));

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(size() * 4);

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    static void swapf(std::string& str, float value) {
      uint32_t x = swap32(*(uint32_t*)(&value));

      str.append((const char*)&x, sizeof(uint32_t));
    }

    static void swapd(std::string& str, double value) {
      uint64_t x = swap64(*(uint64_t*)(&value));

      str.append((const char*)&x, sizeof(uint64_t));
    }

    static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


Parser: do not return from [[noreturn]] Parser::throw_unexpected

#include "hext/Parser.h"

#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/BuiltinCapture.h"
#include "hext/BuiltinMatch.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/ChildCountMatch.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/RegexTest.h"
#include "hext/StringUtil.h"
#include "hext/TextNodeMatch.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "RuleBuilder.h"

#include <string>
#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <boost/optional.hpp>
#include <boost/regex/regex_traits.hpp>
#include <boost/regex/pattern_except.hpp>
#include <gumbo.h>

namespace hext {

/// Convenience macro to store the start of a token. Used within the hext
/// machine definition. Accesses local variables of `Parser::parse()`.
#define TK_START \
  tok_begin = p; \
  tok_end = nullptr;

/// Convenience macro to complete a token. Used within the hext
/// machine defintion. Accesses local variables of `Parser::parse()`.
#define TK_STOP                 \
  assert(tok_begin != nullptr); \
  assert(p != nullptr);         \
  tok_end = p;                  \
  tok = std::string(tok_begin, std::distance(tok_begin, tok_end));

/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel

struct Parser::Impl
{
  Impl(const char * begin, const char * end)
  : p_begin_(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
  }

  Rule parse();

  /// Throw `SyntaxError` with an error message marking an unexpected character.
  [[noreturn]]
  void throw_unexpected() const;

  /// Throw `SyntaxError` with an error message marking an invalid html tag.
  [[noreturn]]
  void throw_invalid_tag(const std::string& tag) const;

  /// Throw `SyntaxError` with an error message marking an invalid regular
  /// expression.
  [[noreturn]]
  void throw_regex_error(
    std::size_t mark_len,
    boost::regex_constants::error_type e_code
  ) const;

  /// Throw `SyntaxError` with an error message complaining about a missing
  /// closing tag.
  [[noreturn]]
  void throw_missing_tag(HtmlTag missing) const;

  /// Throw `SyntaxError` with an error message marking an invalid closing tag.
  ///
  /// \param tag
  ///   The invalid tag name.
  ///
  /// \param expected
  ///   The next expected closing HtmlTag. If empty, a closing tag was
  ///   given but none expected.
  [[noreturn]]
  void throw_unexpected_tag(
    const std::string& tag,
    boost::optional<HtmlTag> expected
  ) const;

  /// Print an error at the current location within hext. Print hext with line
  /// numbers up to the unexpected character.
  ///
  /// \param uc
  ///   A pointer to the unexpected character.
  ///
  /// \param mark_len
  ///   Amount of '^' characters that are used to mark the error location up to
  ///   the unexpected character.
  void print_error_location(
    const char * uc,
    std::size_t mark_len,
    std::ostream& out
  ) const;

  /// The beginning of the hext input.
  const char * p_begin_;

  /// The current character that ragel is processing within the hext input.
  const char * p;

  /// The end of the hext input.
  const char * pe;

  /// The end of the hext input. Same as `Parser::pe`.
  const char * eof;

  /// Ragel's current state.
  int cs;
};

Parser::Parser(const char * begin, const char * end)
: impl_(std::make_unique<Impl>(begin, end))
{
}

Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;
Parser::~Parser() = default;

Rule Parser::parse()
{
  return this->impl_->parse();
}

Rule Parser::Impl::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;

  // Provide shortcut to keep hext-machine's code smaller.
  typedef NthChildMatch::OffsetOf NthOff;

  // In the hext-machine, the rule tree will be constructed with a RuleBuilder.
  RuleBuilder builder;

  // The rule that is currently being built.
  Rule rule;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  // These variables will be accessed by the macros TK_START and TK_STOP.
  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%

  // Throw error if there are missing closing tags.
  if( auto expected_tag = builder.get_expected_tag() )
    this->throw_missing_tag(*expected_tag);

  return std::move(builder.take_rule_tree());
}

void Parser::Impl::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);
  assert(this->p <= this->pe && this->p >= this->p_begin_);

  std::stringstream error_msg;
  if( this->p == this->pe || !this->p )
    error_msg << "Premature termination ";
  else
    error_msg << "Unexpected character '"
              << CharName(*(this->p))
              << "' ";

  if( this->p && this->pe )
    this->print_error_location(this->p, /* mark_len: */ 1, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_invalid_tag(const std::string& tag) const
{
  std::stringstream error_msg;
  error_msg << "Unknown HTML tag '" << tag << "' ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, tag.size(), error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_regex_error(
  std::size_t mark_len,
  boost::regex_constants::error_type e_code
) const
{
  std::stringstream error_msg;
  error_msg << "In regular expression ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  // regex_error::what() not only contains an error message, but also adds the
  // error location. Therefore we use regex_traits::error_string to get a
  // shorter error description.
  boost::regex_traits<boost::regex::value_type> traits;
  error_msg << "\nError: "
            << traits.error_string(e_code);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_missing_tag(HtmlTag missing) const
{
  std::stringstream error_msg;
  error_msg << "Missing closing tag '</"
            << ( missing == HtmlTag::ANY
                 ? "*"
                 : gumbo_normalized_tagname(static_cast<GumboTag>(missing)) )
            << ">' ";

  this->print_error_location(this->pe, /* mark_len: */ 0, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_unexpected_tag(
  const std::string& tag,
  boost::optional<HtmlTag> expected
) const
{
  std::stringstream error_msg;
  error_msg << "Unexpected closing tag '</"
            << tag
            << ">'";

  if( expected )
  {
    error_msg << ", expected '</"
              << ( *expected == HtmlTag::ANY
                   ? "*"
                   : gumbo_normalized_tagname(static_cast<GumboTag>(*expected)) )
              << ">'";
  }

  auto mark_len = tag.size() + 2; // strlen("</")
  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::print_error_location(
  const char * uc,
  std::size_t mark_len,
  std::ostream& out
) const
{
  assert(uc && this->p_begin_ && this->pe);
  assert(uc <= this->pe && uc >= this->p_begin_);
  if( !uc || !this->p_begin_ || !this->pe || uc > this->pe )
    return;

  // The zero-based line and char offset of the unexpected character
  CharPosPair pos = CharPosition(this->p_begin_, uc);

  if( uc == this->pe )
  {
    out << "at end of input:\n\n";
  }
  else
  {
    out << "at line "
        << pos.first + 1 // line_count
        << ", char "
        << pos.second + 1 // char_count
        << ":\n\n";
  }

  // The amount of chars needed to print the biggest line number.
  int number_width = DecimalWidth(static_cast<int>(pos.first + 1));

  // Don't print the unexpected character if it is a newline
  if( uc == this->pe || *uc == '\n' )
    PrintWithLineNumbers(this->p_begin_, uc, number_width, out);
  else
    PrintWithLineNumbers(this->p_begin_, uc + 1, number_width, out);

  if( mark_len < 1 )
    return;

  // If there are non-ascii characters in the input, lazily bail out.
  if( std::any_of(this->p_begin_, this->pe, [](signed char c){return c < 0;}) )
    return;

  // The longest the mark can be is the length of the last line.
  mark_len = std::min(pos.second + 1, static_cast<CharPosType>(mark_len));

  // Print a visual indicator directly under the unexpected token ('^').
  // The required amount of indentation must be known.
  // PrintWithLineNumbers prints lines like this:
  // 1: An SQL query goes into a bar,
  // 2: walks up to two tables and asks,
  // 3: 'Can I join you?'
  std::size_t indent = number_width // chars required to print the line number
                     + 2            // ": "
                     + pos.second+1 // position of the unexpected character from
                                    // the beginning of the line.
                     - mark_len;    // the length of the '^' mark

  out << std::string(indent, ' ')
      << std::string(mark_len, '^')
      << " here\n";
}

} // namespace hext

#include "hext/Parser.h"

#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/BuiltinCapture.h"
#include "hext/BuiltinMatch.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/ChildCountMatch.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/RegexTest.h"
#include "hext/StringUtil.h"
#include "hext/TextNodeMatch.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "RuleBuilder.h"

#include <string>
#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <boost/optional.hpp>
#include <boost/regex/regex_traits.hpp>
#include <boost/regex/pattern_except.hpp>
#include <gumbo.h>

namespace hext {

/// Convenience macro to store the start of a token. Used within the hext
/// machine definition. Accesses local variables of `Parser::parse()`.
#define TK_START \
  tok_begin = p; \
  tok_end = nullptr;

/// Convenience macro to complete a token. Used within the hext
/// machine defintion. Accesses local variables of `Parser::parse()`.
#define TK_STOP                 \
  assert(tok_begin != nullptr); \
  assert(p != nullptr);         \
  tok_end = p;                  \
  tok = std::string(tok_begin, std::distance(tok_begin, tok_end));

/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel

struct Parser::Impl
{
  Impl(const char * begin, const char * end)
  : p_begin_(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
  }

  Rule parse();

  /// Throw `SyntaxError` with an error message marking an unexpected character.
  [[noreturn]]
  void throw_unexpected() const;

  /// Throw `SyntaxError` with an error message marking an invalid html tag.
  [[noreturn]]
  void throw_invalid_tag(const std::string& tag) const;

  /// Throw `SyntaxError` with an error message marking an invalid regular
  /// expression.
  [[noreturn]]
  void throw_regex_error(
    std::size_t mark_len,
    boost::regex_constants::error_type e_code
  ) const;

  /// Throw `SyntaxError` with an error message complaining about a missing
  /// closing tag.
  [[noreturn]]
  void throw_missing_tag(HtmlTag missing) const;

  /// Throw `SyntaxError` with an error message marking an invalid closing tag.
  ///
  /// \param tag
  ///   The invalid tag name.
  ///
  /// \param expected
  ///   The next expected closing HtmlTag. If empty, a closing tag was
  ///   given but none expected.
  [[noreturn]]
  void throw_unexpected_tag(
    const std::string& tag,
    boost::optional<HtmlTag> expected
  ) const;

  /// Print an error at the current location within hext. Print hext with line
  /// numbers up to the unexpected character.
  ///
  /// \param uc
  ///   A pointer to the unexpected character.
  ///
  /// \param mark_len
  ///   Amount of '^' characters that are used to mark the error location up to
  ///   the unexpected character.
  void print_error_location(
    const char * uc,
    std::size_t mark_len,
    std::ostream& out
  ) const;

  /// The beginning of the hext input.
  const char * p_begin_;

  /// The current character that ragel is processing within the hext input.
  const char * p;

  /// The end of the hext input.
  const char * pe;

  /// The end of the hext input. Same as `Parser::pe`.
  const char * eof;

  /// Ragel's current state.
  int cs;
};

Parser::Parser(const char * begin, const char * end)
: impl_(std::make_unique<Impl>(begin, end))
{
}

Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;
Parser::~Parser() = default;

Rule Parser::parse()
{
  return this->impl_->parse();
}

Rule Parser::Impl::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;

  // Provide shortcut to keep hext-machine's code smaller.
  typedef NthChildMatch::OffsetOf NthOff;

  // In the hext-machine, the rule tree will be constructed with a RuleBuilder.
  RuleBuilder builder;

  // The rule that is currently being built.
  Rule rule;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  // These variables will be accessed by the macros TK_START and TK_STOP.
  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%

  // Throw error if there are missing closing tags.
  if( auto expected_tag = builder.get_expected_tag() )
    this->throw_missing_tag(*expected_tag);

  return std::move(builder.take_rule_tree());
}

void Parser::Impl::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);
  assert(this->p <= this->pe && this->p >= this->p_begin_);

  if( !this->p || !this->pe )
    return;

  std::stringstream error_msg;
  if( this->p == this->pe )
  {
    error_msg << "Premature termination ";
  }
  else
  {
    error_msg << "Unexpected character '"
              << CharName(*(this->p))
              << "' ";
  }

  this->print_error_location(this->p, /* mark_len: */ 1, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_invalid_tag(const std::string& tag) const
{
  std::stringstream error_msg;
  error_msg << "Unknown HTML tag '" << tag << "' ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, tag.size(), error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_regex_error(
  std::size_t mark_len,
  boost::regex_constants::error_type e_code
) const
{
  std::stringstream error_msg;
  error_msg << "In regular expression ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  // regex_error::what() not only contains an error message, but also adds the
  // error location. Therefore we use regex_traits::error_string to get a
  // shorter error description.
  boost::regex_traits<boost::regex::value_type> traits;
  error_msg << "\nError: "
            << traits.error_string(e_code);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_missing_tag(HtmlTag missing) const
{
  std::stringstream error_msg;
  error_msg << "Missing closing tag '</"
            << ( missing == HtmlTag::ANY
                 ? "*"
                 : gumbo_normalized_tagname(static_cast<GumboTag>(missing)) )
            << ">' ";

  this->print_error_location(this->pe, /* mark_len: */ 0, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::throw_unexpected_tag(
  const std::string& tag,
  boost::optional<HtmlTag> expected
) const
{
  std::stringstream error_msg;
  error_msg << "Unexpected closing tag '</"
            << tag
            << ">'";

  if( expected )
  {
    error_msg << ", expected '</"
              << ( *expected == HtmlTag::ANY
                   ? "*"
                   : gumbo_normalized_tagname(static_cast<GumboTag>(*expected)) )
              << ">'";
  }

  auto mark_len = tag.size() + 2; // strlen("</")
  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::Impl::print_error_location(
  const char * uc,
  std::size_t mark_len,
  std::ostream& out
) const
{
  assert(uc && this->p_begin_ && this->pe);
  assert(uc <= this->pe && uc >= this->p_begin_);
  if( !uc || !this->p_begin_ || !this->pe || uc > this->pe )
    return;

  // The zero-based line and char offset of the unexpected character
  CharPosPair pos = CharPosition(this->p_begin_, uc);

  if( uc == this->pe )
  {
    out << "at end of input:\n\n";
  }
  else
  {
    out << "at line "
        << pos.first + 1 // line_count
        << ", char "
        << pos.second + 1 // char_count
        << ":\n\n";
  }

  // The amount of chars needed to print the biggest line number.
  int number_width = DecimalWidth(static_cast<int>(pos.first + 1));

  // Don't print the unexpected character if it is a newline
  if( uc == this->pe || *uc == '\n' )
    PrintWithLineNumbers(this->p_begin_, uc, number_width, out);
  else
    PrintWithLineNumbers(this->p_begin_, uc + 1, number_width, out);

  if( mark_len < 1 )
    return;

  // If there are non-ascii characters in the input, lazily bail out.
  if( std::any_of(this->p_begin_, this->pe, [](signed char c){return c < 0;}) )
    return;

  // The longest the mark can be is the length of the last line.
  mark_len = std::min(pos.second + 1, static_cast<CharPosType>(mark_len));

  // Print a visual indicator directly under the unexpected token ('^').
  // The required amount of indentation must be known.
  // PrintWithLineNumbers prints lines like this:
  // 1: An SQL query goes into a bar,
  // 2: walks up to two tables and asks,
  // 3: 'Can I join you?'
  std::size_t indent = number_width // chars required to print the line number
                     + 2            // ": "
                     + pos.second+1 // position of the unexpected character from
                                    // the beginning of the line.
                     - mark_len;    // the length of the '^' mark

  out << std::string(indent, ' ')
      << std::string(mark_len, '^')
      << " here\n";
}

} // namespace hext