VALUE -> TNETS_T

VALUE -> TNETS_T

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    do {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    } while(sub_size > 0);

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    TNETS_T arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    do {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    } while(sub_size > 0);

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    VALUE arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}


Split up body state machine

Split up body state machine

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = EMPTY_BUFFER.dup
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    else
      content_length = nil
    end
  }

  action consume_null {
    buffer.bytesize < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.bytesize < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  consume_body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer);
  body = consume_body >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      EMPTY_BUFFER = "".force_encoding("BINARY").freeze
      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size # how many bytes current message contains in total
        content_length = state.content_length # content length of current message

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end

%%{
  machine message;

  # data, p, pe, eof, cs, top, stack, ts, te and act
  getkey data.getbyte(p); # code for retrieving current char

  ## Action state - needs resetting once consumed!
  action mark {
    buffer = EMPTY_BUFFER.dup
  }
  action buffer {
    buffer << fc
  }
  action mark_key {
    mk = buffer # needs reset
    buffer = nil
  }
  action mark_message {
    message = Stomp::Message.new(nil, nil)
    message_size = 0
  }
  action check_message_size {
    message_size += 1
    raise MessageSizeExceeded if message_size > max_message_size
  }

  ## Action commands - should reset used state!
  action write_command {
    message.write_command(buffer)
    buffer = nil
  }

  action write_header {
    message.write_header(mk, buffer)
    mk = buffer = nil
  }

  action write_body {
    message.write_body(buffer)
    buffer = nil
  }

  action finish_headers {
    if message.headers.has_key?("content-length")
      content_length = Integer(message.headers["content-length"])
    else
      content_length = nil
    end
  }

  action consume_null {
    buffer.bytesize < content_length if content_length
  }

  action consume_octet {
    if content_length
      buffer.bytesize < content_length
    else
      true
    end
  }

  action finish_message {
    yield message
    message = nil
  }

  ## Stomp message grammar

  NULL = "\0";
  EOL = "\r"? . "\n";
  OCTET = any;

  client_command = "CONNECT" > mark;
  command = client_command $ buffer % write_command . EOL;

  HEADER_ESCAPE = "\\" . ("\\" | "n" | "r" | "c");
  HEADER_OCTET = HEADER_ESCAPE | (OCTET - "\r" - "\n" - "\\" - ":");
  header_key = HEADER_OCTET+ > mark $ buffer % mark_key;
  header_value = HEADER_OCTET* > mark $ buffer;
  header = header_key . ":" . header_value;
  headers = (header % write_header . EOL)* % finish_headers . EOL;

  body = ((NULL when consume_null | ^NULL when consume_octet)* $ buffer) >to(mark) % write_body <: NULL;

  message = ((command > mark_message) :> headers :> (body @ finish_message)) $ check_message_size;

  stream := (EOL | message)*;
}%%

module Stompede
  module Stomp
    # Provides an API for running parsers.
    #
    # It:
    # - provides a .parse method which remembers state between invocations
    # - buffering of data between chunks
    class Parser
      # this manipulates the singleton class of our context,
      # so we do not want to run this code very often or we
      # bust our ruby method caching
      %% write data noprefix;

      class << self
        # @attr [Integer] maximum size (in bytes) a message may become before raising MessageSizeExceeded.
        attr_accessor :max_message_size
      end

      EMPTY_BUFFER = "".force_encoding("BINARY").freeze
      self.max_message_size = 1024 * 100 # 100KB

      # Parse a chunk of Stomp-formatted data into a Message.
      #
      # @param [String] data
      # @param [Parser] state
      # @return [Stomp::Message, nil]
      def self.parse(data, state)
        pe = data.bytesize # end of chunk

        p = 0 # pointer to current character
        message = state.message # message currently being parsed, if any
        cs = state.current_state # current state
        mk = state.mark_key # key for header currently being read
        buffer = state.buffer # buffered data for marks
        message_size = state.message_size # how many bytes current message contains in total
        content_length = state.content_length # content length of current message

        %% write exec;

        if cs == Stomp::Parser.error
          # build error message context
          ctx = 7
          min = [0, p - ctx].max
          cur = p - min
          err = data.byteslice(min, ctx + 1 + ctx)
          chr = err[cur]
          err[cur] = " -->#{err[cur]}<-- "

          raise ParseError.new("unexpected #{chr.inspect} in data (#{err.inspect})")
        else
          state.message = message
          state.current_state = cs
          state.mark_key = mk
          state.buffer = buffer
          state.message_size = message_size
          state.content_length = content_length
        end

        nil
      end

      # Construct the parser.
      def initialize
        @error = nil
        @buffer = nil
        @message_size = nil
        @current_state = Stomp::Parser.start
        @message = nil
        @mark_key = nil
      end

      # @return [StandardError] error raised during parsing
      attr_accessor :error

      # @return [Integer] message size accumulated so far
      attr_accessor :message_size

      # @return [String] binary string of current parsing buffer
      attr_accessor :buffer

      # @return [Integer, nil]
      attr_accessor :content_length

      # @return [Integer] current parsing state
      attr_accessor :current_state

      # @return [Stomp::Message] stomp message currently being parsed
      attr_accessor :message

      # @return [String] header key currently being parsed
      attr_accessor :mark_key

      # Parse a chunk of data. Retains state beteween invocations.
      #
      # @param [String] data
      # @raise [ParseError]
      def parse(data)
        raise error if error

        begin
          Parser.parse(data, self) do |message|
            yield message
          end
        rescue => error
          self.error = error
          raise
        end
      end
    end
  end
end


Allow 0X-prefixed hex numbers (since I support E and P...)

//
//  yl_parser_ragel.rl
//
//  Created by Edmund Kapusniak on 14/03/2014.
//  Copyright (c) 2014 Edmund Kapusniak. All rights reserved.
//

/*
    ragel yl_parser_ragel.rl -o yl_parser_ragel.cpp -G2
*/

#include "yl_parser.h"
#include <stdint.h>
#include <intformat.h>
#include <unordered_map>
#include <region.h>
#include "yl_token.h"
#include "yl_diagnostics.h"

#pragma GCC diagnostic ignored "-Wunreachable-code"

void* XecParseAlloc( void* (*malloc)( size_t ) );
void  XecParse( void* yyp, int yymajor, yl_token* yyminor, yl_parser* p );
void  XecParseFree( void* p, void (*free)( void* ) );

%%{

    machine lexer;

    alphtype unsigned char;

    action nl { diagnostics->newline( (int)( offset + ( p - buffer ) - 1 ) ); }
    action ts { sloc = (int)( offset + ( p - buffer ) ); data.clear(); }
    action dc { data.append( (char)fc ); }

    action neof
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        diagnostics->error( sloc, "unexpected end of file" );
        fbreak;
    }

    action nchar
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        if ( fc >= 0x20 && fc <= 0x7E )
            diagnostics->error( sloc, "unexpected character '%c'", fc );
        else
            diagnostics->error( sloc, "unexpected character '\\x%02X'", fc );
        if ( diagnostics->error_count() >= ERROR_LIMIT )
            goto error;
    }

    # Newlines (handling CR LF properly), comments, and whitespace.

    newline         = ( '\n' | '\r' | '\r\n' ) $1 %0 %nl %!nl ;
    block_comment   = '/*' ( any | newline )* :>> '*/' ;
    line_comment    = '//' any* :>> newline ;
    whitespace      = [ \t] | newline | block_comment | line_comment ;

    identifier =
    (
            [a-zA-Z_][a-zA-Z0-9_]*
    ) $dc;

    number =
    (
            ( '0' | [1-9][0-9]* ) ( '.' [0-9]+ )? ( [eE] ( '+' | '-' )? [0-9]+ )?
        |   '0' [xX] [0-9A-Fa-f]+ ( '.' [0-9A-Fa-f]+ )? ( [pP] ( '+' | '-' )? [0-9]+ )?
    ) $dc;

    string_escape =
    (
            '\\"'   @ { data.append( '\"' ); }
        |   '\\\\'  @ { data.append( '\\' ); }
        |   '\\/'   @ { data.append( '/' ); }
        |   '\\b'   @ { data.append( '\b' ); }
        |   '\\f'   @ { data.append( '\f' ); }
        |   '\\n'   @ { data.append( '\n' ); }
        |   '\\r'   @ { data.append( '\r' ); }
        |   '\\t'   @ { data.append( '\t' ); }
        |   '\\x'
            ( [0-9] @ { temp = ( fc - '0' ) << 4; }
            | [a-f] @ { temp = ( fc - 'a' + 0x0A ) << 4; }
            | [A-F] @ { temp = ( fc - 'A' + 0x0A ) << 4; }
            )
            ( [0-9] @ { data.append( (char)( temp | fc - '0' ) ); }
            | [a-f] @ { data.append( (char)( temp | fc - 'a' + 0x0A ) ); }
            | [A-F] @ { data.append( (char)( temp | fc - 'A' + 0x0A ) ); }
            )
        |   '\\u'
            ( [0-9] @ { temp = ( fc - '0' ) << 20; }
            | [a-f] @ { temp = ( fc - 'a' + 0x0A ) << 20; }
            | [A-F] @ { temp = ( fc - 'A' + 0x0A ) << 20; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 16; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 16; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 16; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 12; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 12; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 12; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 8; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 8; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 8; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 4; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 4; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 4; }
            )
            ( [0-9] @ { temp |= fc - '0'; }
            | [a-f] @ { temp |= fc - 'a' + 0x0A; }
            | [A-F] @ { temp |= fc - 'A' + 0x0A; }
            )
                @
                {
                    if ( ! encode_utf8( &data, temp ) )
                    {
                        diagnostics->error(
                                sloc, "invalid codepoint U+%04" PRIX32, temp );
                        if ( diagnostics->error_count() >= ERROR_LIMIT )
                            goto error;
                    }
                }
    );

    string_char =
    (
            any - [\n\r\"\\]
    ) $dc;

    string_quoted =
    (
            '"' string_char* ( string_escape string_char* )* '"'
    );

    string =
    (
            ( string_quoted whitespace* )+
    );

    main :=
    (
            whitespace

        |   identifier >ts
                %
                {
                    yl_token* token = make_identifier( sloc, &data );
                    TOKEN( token );
                }

        |   number >ts
                %
                {
                    size_t length = data.size();
                    data.append( '\0' );
                    const char* number = (const char*)data.tearoff();
                    MTOKEN( YL_TOKEN_NUMBER, sloc, number, length );
                }

        |   string >ts
                %
                {
                    size_t length = data.size();
                    data.append( '\0' );
                    const char* string = (const char*)data.tearoff();
                    MTOKEN( YL_TOKEN_STRING, sloc, string, length );
                }

        |   '!'    >ts %{ MTOKEN( YL_TOKEN_XMARK, sloc, "!", 1 ); }
        |   '%'    >ts %{ MTOKEN( YL_TOKEN_PERCENT, sloc, "%", 1 ); }
        |   '&'    >ts %{ MTOKEN( YL_TOKEN_AMPERSAND, sloc, "&", 1 ); }
        |   '('    >ts %{ MTOKEN( YL_TOKEN_LPN, sloc, "(", 1 ); }
        |   ')'    >ts %{ MTOKEN( YL_TOKEN_RPN, sloc, ")", 1 ); }
        |   '*'    >ts %{ MTOKEN( YL_TOKEN_ASTERISK, sloc, "*", 1 ); }
        |   '+'    >ts %{ MTOKEN( YL_TOKEN_PLUS, sloc, "+", 1 ); }
        |   ','    >ts %{ MTOKEN( YL_TOKEN_COMMA, sloc, ",", 1 ); }
        |   '-'    >ts %{ MTOKEN( YL_TOKEN_MINUS, sloc, "-", 1 ); }
        |   '.'    >ts %{ MTOKEN( YL_TOKEN_PERIOD, sloc, ".", 1 ); }
        |   '/'    >ts %{ MTOKEN( YL_TOKEN_SOLIDUS, sloc, "/", 1 ); }
        |   ':'    >ts %{ MTOKEN( YL_TOKEN_COLON, sloc, ":", 1 ); }
        |   ';'    >ts %{ MTOKEN( YL_TOKEN_SEMICOLON, sloc, ";", 1 ); }
        |   '<'    >ts %{ MTOKEN( YL_TOKEN_LESS, sloc, "<", 1 ); }
        |   '='    >ts %{ MTOKEN( YL_TOKEN_ASSIGN, sloc, "=", 1 ); }
        |   '>'    >ts %{ MTOKEN( YL_TOKEN_GREATER, sloc, ">", 1 ); }
        |   '?'    >ts %{ MTOKEN( YL_TOKEN_QMARK, sloc, "?", 1 ); }
        |   '['    >ts %{ MTOKEN( YL_TOKEN_LSQ, sloc, "[", 1 ); }
        |   ']'    >ts %{ MTOKEN( YL_TOKEN_RSQ, sloc, "]", 1 ); }
        |   '^'    >ts %{ MTOKEN( YL_TOKEN_CARET, sloc, "^", 1 ); }
        |   '{'    >ts %{ MTOKEN( YL_TOKEN_LBR, sloc, "{", 1 ); }
        |   '|'    >ts %{ MTOKEN( YL_TOKEN_VBAR, sloc, "|", 1 ); }
        |   '}'    >ts %{ MTOKEN( YL_TOKEN_RBR, sloc, "}", 1 ); }
        |   '~'    >ts %{ MTOKEN( YL_TOKEN_TILDE, sloc, "~", 1 ); }

        |   '++'   >ts %{ MTOKEN( YL_TOKEN_INCREMENT, sloc, "++", 2 ); }
        |   '--'   >ts %{ MTOKEN( YL_TOKEN_DECREMENT, sloc, "--", 2 ); }

        |   '..'   >ts %{ MTOKEN( YL_TOKEN_CONCATENATE, sloc, "..", 2 ); }

        |   '<<'   >ts %{ MTOKEN( YL_TOKEN_LSHIFT, sloc, "<<", 2 ); }
        |   '>>'   >ts %{ MTOKEN( YL_TOKEN_LRSHIFT, sloc, ">>", 2 ); }
        |   '~>>'  >ts %{ MTOKEN( YL_TOKEN_ARSHIFT, sloc, "~>>", 3 ); }

        |   '!='   >ts %{ MTOKEN( YL_TOKEN_NOTEQUAL, sloc, "!=", 2 ); }
        |   '<='   >ts %{ MTOKEN( YL_TOKEN_LESSEQUAL, sloc, "<=", 2 ); }
        |   '=='   >ts %{ MTOKEN( YL_TOKEN_EQUAL, sloc, "==", 2 ); }
        |   '>='   >ts %{ MTOKEN( YL_TOKEN_GREATEREQUAL, sloc, ">=", 2 ); }
        |   '!in'  >ts %{ MTOKEN( YL_TOKEN_NOTIN, sloc, "!in", 3 ); }
        |   '!is'  >ts %{ MTOKEN( YL_TOKEN_NOTIS, sloc, "!is", 3 ); }

        |   '%='   >ts %{ MTOKEN( YL_TOKEN_MODASSIGN, sloc, "%=", 2 ); }
        |   '&='   >ts %{ MTOKEN( YL_TOKEN_BITANDASSIGN, sloc, "&=", 2 ); }
        |   '*='   >ts %{ MTOKEN( YL_TOKEN_MULASSIGN, sloc, "*=", 2 ); }
        |   '+='   >ts %{ MTOKEN( YL_TOKEN_ADDASSIGN, sloc, "+=", 2 ); }
        |   '-='   >ts %{ MTOKEN( YL_TOKEN_SUBASSIGN, sloc, "-=", 2 ); }
        |   '/='   >ts %{ MTOKEN( YL_TOKEN_DIVASSIGN, sloc, "/=", 2 ); }
        |   '^='   >ts %{ MTOKEN( YL_TOKEN_BITXORASSIGN, sloc, "^=", 2 ); }
        |   '|='   >ts %{ MTOKEN( YL_TOKEN_BITORASSIGN, sloc, "|=", 2 ); }
        |   '~='   >ts %{ MTOKEN( YL_TOKEN_INTDIVASSIGN, sloc, "~=", 2 ); }
        |   '<<='  >ts %{ MTOKEN( YL_TOKEN_LSHIFTASSIGN, sloc, "<<=", 3 ); }
        |   '>>='  >ts %{ MTOKEN( YL_TOKEN_LRSHIFTASSIGN, sloc, ">>=", 3 ); }
        |   '~>>=' >ts %{ MTOKEN( YL_TOKEN_ARSHIFTASSIGN, sloc, "~>>=", 4 ); }

        |   '&&'   >ts %{ MTOKEN( YL_TOKEN_LOGICAND, sloc, "&&", 2 ); }
        |   '^^'   >ts %{ MTOKEN( YL_TOKEN_LOGICXOR, sloc, "^^", 2 ); }
        |   '||'   >ts %{ MTOKEN( YL_TOKEN_LOGICOR, sloc, "||", 2 ); }

        |   '::'   >ts %{ MTOKEN( YL_TOKEN_EACHKEY, sloc, "::", 2 ); }
        |   '...'  >ts %{ MTOKEN( YL_TOKEN_ELLIPSIS, sloc, "...", 3 ); }

    )**

    @/neof

    $!nchar $!{ fgoto main; };

}%%

%% write data;

template < typename ... arguments_t >
yl_token* yl_parser::make_token( arguments_t ... arguments )
{
    void* p = nullptr;
    if ( recycle_tokens.size() )
    {
        p = recycle_tokens.back();
        recycle_tokens.pop_back();
    }
    else
    {
        p = malloc( sizeof( yl_token ) );
    }
    return new ( p ) yl_token( arguments ... );
}

struct yl_keywords
    :   public std::unordered_map< symkey, yl_token_kind >
{

    yl_keywords()
    {
        emplace( symkey( "break" ),     YL_KEYWORD_BREAK );
        emplace( symkey( "case" ),      YL_KEYWORD_CASE );
        emplace( symkey( "catch" ),     YL_KEYWORD_CATCH );
        emplace( symkey( "continue" ),  YL_KEYWORD_CONTINUE );
        emplace( symkey( "def" ),       YL_KEYWORD_DEF );
        emplace( symkey( "default" ),   YL_KEYWORD_DEFAULT );
        emplace( symkey( "delete" ),    YL_KEYWORD_DELETE );
        emplace( symkey( "do" ),        YL_KEYWORD_DO );
        emplace( symkey( "else" ),      YL_KEYWORD_ELSE );
        emplace( symkey( "false" ),     YL_KEYWORD_FALSE );
        emplace( symkey( "finally" ),   YL_KEYWORD_FINALLY );
        emplace( symkey( "for" ),       YL_KEYWORD_FOR );
        emplace( symkey( "if" ),        YL_KEYWORD_IF );
        emplace( symkey( "new" ),       YL_KEYWORD_NEW );
        emplace( symkey( "null" ),      YL_KEYWORD_NULL );
        emplace( symkey( "return" ),    YL_KEYWORD_RETURN );
        emplace( symkey( "superof" ),   YL_KEYWORD_SUPEROF );
        emplace( symkey( "switch" ),    YL_KEYWORD_SWITCH );
        emplace( symkey( "throw" ),     YL_KEYWORD_THROW );
        emplace( symkey( "true" ),      YL_KEYWORD_TRUE );
        emplace( symkey( "try" ),       YL_KEYWORD_TRY );
        emplace( symkey( "using" ),     YL_KEYWORD_USING );
        emplace( symkey( "var" ),       YL_KEYWORD_VAR );
        emplace( symkey( "while" ),     YL_KEYWORD_WHILE );
        emplace( symkey( "yield" ),     YL_KEYWORD_YIELD );
        emplace( symkey( "in" ),        YL_TOKEN_IN );
        emplace( symkey( "is" ),        YL_TOKEN_IS );
    }

};

yl_token* yl_parser::make_identifier( int sloc, region_buffer* data )
{
    static yl_keywords keywords;

    size_t length = data->size();
    const char* identifier = (const char*)data->get();
    symkey key( identifier, length );

    // Check for keyword.
    auto i = keywords.find( key );
    if ( i != keywords.end() )
    {
        return make_token( i->second, sloc, i->first.c_str(), i->first.size() );
    }

    // Check for existing identifier.
    auto j = identifiers.find( key );
    if ( j != identifiers.end() )
    {
        return make_token( YL_TOKEN_IDENTIFIER, sloc, j->c_str(), j->size() );
    }

    // New identifier.
    data->append( '\0' );
    identifier = (const char*)data->tearoff();
    identifiers.emplace( key.hash(), identifier, length );
    return make_token( YL_TOKEN_IDENTIFIER, sloc, identifier, length );

}

static bool encode_utf8( region_buffer* data, uint32_t cp )
{
    if ( cp <= 0x7F )
    {
        data->append( (uint8_t)cp );
        return true;
    }
    else if ( cp <= 0x7FF )
    {
        data->append( (uint8_t)( 0xC0 | ( cp >> 6 & 0x1F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else if ( cp <= 0xFFFF )
    {
        data->append( (uint8_t)( 0xE0 | ( cp >> 12 & 0x0F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 6 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else if ( cp <= 0x10FFFF )
    {
        data->append( (uint8_t)( 0xF0 | ( cp >> 18 & 0x07 ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 12 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 6 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else
    {
        return false;
    }
}

bool yl_parser::parse( const char* path )
{
    region_scope rscope( root->alloc );
    region_buffer data( root->alloc );

    // Push function scope.
    scopes.push_back( root->function->scope );

    // Open file.
    diagnostics->set_filename( path );
    FILE* file = fopen( path, "r" );
    if ( ! file )
    {
        diagnostics->error( 0, "unable to open file" );
        return false;
    }

    // Name of root function.
    root->function->funcname = root->alloc.strdup( path );

    // File state.
    int offset = 0;
    unsigned char* buffer = (unsigned char*)malloc( BUFFER_SIZE );

    // Token state.
    int sloc = -1;
    uint32_t temp = 0;

    // Parser state.
    void* parser = XecParseAlloc( malloc );

    // Initialize state machine.
    int cs;

    %% write init;

    // Perform lexing.
#define TOKEN( token ) \
    { \
        XecParse( parser, token->kind, token, this ); \
        if ( diagnostics->error_count() >= ERROR_LIMIT ) \
            goto error; \
    }
#define MTOKEN( ... ) \
    { \
        yl_token* token = make_token( __VA_ARGS__ ); \
        XecParse( parser, token->kind, token, this ); \
        if ( diagnostics->error_count() >= ERROR_LIMIT ) \
            goto error; \
    }

    bool iseof = false;
    while ( ! iseof )
    {
        size_t read = fread( buffer, 1, BUFFER_SIZE, file );
        iseof = read < BUFFER_SIZE;

        if ( iseof && ferror( file ) )
        {
            data.shrink();
            diagnostics->error( offset, "error reading file" );
            goto error;
        }

        const unsigned char* p      = buffer;
        const unsigned char* pe     = buffer + read;
        const unsigned char* eof    = iseof ? pe : nullptr;

        %% write exec;

        offset += read;
    }

    if ( ! ferror( file ) )
    {
        MTOKEN( YL_TOKEN_EOF, offset, "", 0 );
    }

#undef MTOKEN
#undef TOKEN

error:

    // Free parser.
    XecParseFree( parser, free );

    // Close file.
    free( buffer );
    fclose( file );

    // Check if there were errors.
    return diagnostics->error_count() == 0;
}

#pragma unused( lexer_first_final )
#pragma unused( lexer_error )
#pragma unused( lexer_en_main )

//
//  yl_parser_ragel.rl
//
//  Created by Edmund Kapusniak on 14/03/2014.
//  Copyright (c) 2014 Edmund Kapusniak. All rights reserved.
//

/*
    ragel yl_parser_ragel.rl -o yl_parser_ragel.cpp -G2
*/

#include "yl_parser.h"
#include <stdint.h>
#include <intformat.h>
#include <unordered_map>
#include <region.h>
#include "yl_token.h"
#include "yl_diagnostics.h"

#pragma GCC diagnostic ignored "-Wunreachable-code"

void* XecParseAlloc( void* (*malloc)( size_t ) );
void  XecParse( void* yyp, int yymajor, yl_token* yyminor, yl_parser* p );
void  XecParseFree( void* p, void (*free)( void* ) );

%%{

    machine lexer;

    alphtype unsigned char;

    action nl { diagnostics->newline( (int)( offset + ( p - buffer ) - 1 ) ); }
    action ts { sloc = (int)( offset + ( p - buffer ) ); data.clear(); }
    action dc { data.append( (char)fc ); }

    action neof
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        diagnostics->error( sloc, "unexpected end of file" );
        fbreak;
    }

    action nchar
    {
        data.shrink();
        int sloc = (int)( offset + ( p - buffer ) );
        if ( fc >= 0x20 && fc <= 0x7E )
            diagnostics->error( sloc, "unexpected character '%c'", fc );
        else
            diagnostics->error( sloc, "unexpected character '\\x%02X'", fc );
        if ( diagnostics->error_count() >= ERROR_LIMIT )
            goto error;
    }

    # Newlines (handling CR LF properly), comments, and whitespace.

    newline         = ( '\n' | '\r' | '\r\n' ) $1 %0 %nl %!nl ;
    block_comment   = '/*' ( any | newline )* :>> '*/' ;
    line_comment    = '//' any* :>> newline ;
    whitespace      = [ \t] | newline | block_comment | line_comment ;

    identifier =
    (
            [a-zA-Z_][a-zA-Z0-9_]*
    ) $dc;

    number =
    (
            ( '0' | [1-9][0-9]* ) ( '.' [0-9]+ )? ( [eE] ( '+' | '-' )? [0-9]+ )?
        |   '0x' [0-9A-Fa-f]+ ( '.' [0-9A-Fa-f]+ )? ( [pP] ( '+' | '-' )? [0-9]+ )?
    ) $dc;

    string_escape =
    (
            '\\"'   @ { data.append( '\"' ); }
        |   '\\\\'  @ { data.append( '\\' ); }
        |   '\\/'   @ { data.append( '/' ); }
        |   '\\b'   @ { data.append( '\b' ); }
        |   '\\f'   @ { data.append( '\f' ); }
        |   '\\n'   @ { data.append( '\n' ); }
        |   '\\r'   @ { data.append( '\r' ); }
        |   '\\t'   @ { data.append( '\t' ); }
        |   '\\x'
            ( [0-9] @ { temp = ( fc - '0' ) << 4; }
            | [a-f] @ { temp = ( fc - 'a' + 0x0A ) << 4; }
            | [A-F] @ { temp = ( fc - 'A' + 0x0A ) << 4; }
            )
            ( [0-9] @ { data.append( (char)( temp | fc - '0' ) ); }
            | [a-f] @ { data.append( (char)( temp | fc - 'a' + 0x0A ) ); }
            | [A-F] @ { data.append( (char)( temp | fc - 'A' + 0x0A ) ); }
            )
        |   '\\u'
            ( [0-9] @ { temp = ( fc - '0' ) << 20; }
            | [a-f] @ { temp = ( fc - 'a' + 0x0A ) << 20; }
            | [A-F] @ { temp = ( fc - 'A' + 0x0A ) << 20; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 16; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 16; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 16; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 12; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 12; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 12; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 8; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 8; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 8; }
            )
            ( [0-9] @ { temp |= ( fc - '0' ) << 4; }
            | [a-f] @ { temp |= ( fc - 'a' + 0x0A ) << 4; }
            | [A-F] @ { temp |= ( fc - 'A' + 0x0A ) << 4; }
            )
            ( [0-9] @ { temp |= fc - '0'; }
            | [a-f] @ { temp |= fc - 'a' + 0x0A; }
            | [A-F] @ { temp |= fc - 'A' + 0x0A; }
            )
                @
                {
                    if ( ! encode_utf8( &data, temp ) )
                    {
                        diagnostics->error(
                                sloc, "invalid codepoint U+%04" PRIX32, temp );
                        if ( diagnostics->error_count() >= ERROR_LIMIT )
                            goto error;
                    }
                }
    );

    string_char =
    (
            any - [\n\r\"\\]
    ) $dc;

    string_quoted =
    (
            '"' string_char* ( string_escape string_char* )* '"'
    );

    string =
    (
            ( string_quoted whitespace* )+
    );

    main :=
    (
            whitespace

        |   identifier >ts
                %
                {
                    yl_token* token = make_identifier( sloc, &data );
                    TOKEN( token );
                }

        |   number >ts
                %
                {
                    size_t length = data.size();
                    data.append( '\0' );
                    const char* number = (const char*)data.tearoff();
                    MTOKEN( YL_TOKEN_NUMBER, sloc, number, length );
                }

        |   string >ts
                %
                {
                    size_t length = data.size();
                    data.append( '\0' );
                    const char* string = (const char*)data.tearoff();
                    MTOKEN( YL_TOKEN_STRING, sloc, string, length );
                }

        |   '!'    >ts %{ MTOKEN( YL_TOKEN_XMARK, sloc, "!", 1 ); }
        |   '%'    >ts %{ MTOKEN( YL_TOKEN_PERCENT, sloc, "%", 1 ); }
        |   '&'    >ts %{ MTOKEN( YL_TOKEN_AMPERSAND, sloc, "&", 1 ); }
        |   '('    >ts %{ MTOKEN( YL_TOKEN_LPN, sloc, "(", 1 ); }
        |   ')'    >ts %{ MTOKEN( YL_TOKEN_RPN, sloc, ")", 1 ); }
        |   '*'    >ts %{ MTOKEN( YL_TOKEN_ASTERISK, sloc, "*", 1 ); }
        |   '+'    >ts %{ MTOKEN( YL_TOKEN_PLUS, sloc, "+", 1 ); }
        |   ','    >ts %{ MTOKEN( YL_TOKEN_COMMA, sloc, ",", 1 ); }
        |   '-'    >ts %{ MTOKEN( YL_TOKEN_MINUS, sloc, "-", 1 ); }
        |   '.'    >ts %{ MTOKEN( YL_TOKEN_PERIOD, sloc, ".", 1 ); }
        |   '/'    >ts %{ MTOKEN( YL_TOKEN_SOLIDUS, sloc, "/", 1 ); }
        |   ':'    >ts %{ MTOKEN( YL_TOKEN_COLON, sloc, ":", 1 ); }
        |   ';'    >ts %{ MTOKEN( YL_TOKEN_SEMICOLON, sloc, ";", 1 ); }
        |   '<'    >ts %{ MTOKEN( YL_TOKEN_LESS, sloc, "<", 1 ); }
        |   '='    >ts %{ MTOKEN( YL_TOKEN_ASSIGN, sloc, "=", 1 ); }
        |   '>'    >ts %{ MTOKEN( YL_TOKEN_GREATER, sloc, ">", 1 ); }
        |   '?'    >ts %{ MTOKEN( YL_TOKEN_QMARK, sloc, "?", 1 ); }
        |   '['    >ts %{ MTOKEN( YL_TOKEN_LSQ, sloc, "[", 1 ); }
        |   ']'    >ts %{ MTOKEN( YL_TOKEN_RSQ, sloc, "]", 1 ); }
        |   '^'    >ts %{ MTOKEN( YL_TOKEN_CARET, sloc, "^", 1 ); }
        |   '{'    >ts %{ MTOKEN( YL_TOKEN_LBR, sloc, "{", 1 ); }
        |   '|'    >ts %{ MTOKEN( YL_TOKEN_VBAR, sloc, "|", 1 ); }
        |   '}'    >ts %{ MTOKEN( YL_TOKEN_RBR, sloc, "}", 1 ); }
        |   '~'    >ts %{ MTOKEN( YL_TOKEN_TILDE, sloc, "~", 1 ); }

        |   '++'   >ts %{ MTOKEN( YL_TOKEN_INCREMENT, sloc, "++", 2 ); }
        |   '--'   >ts %{ MTOKEN( YL_TOKEN_DECREMENT, sloc, "--", 2 ); }

        |   '..'   >ts %{ MTOKEN( YL_TOKEN_CONCATENATE, sloc, "..", 2 ); }

        |   '<<'   >ts %{ MTOKEN( YL_TOKEN_LSHIFT, sloc, "<<", 2 ); }
        |   '>>'   >ts %{ MTOKEN( YL_TOKEN_LRSHIFT, sloc, ">>", 2 ); }
        |   '~>>'  >ts %{ MTOKEN( YL_TOKEN_ARSHIFT, sloc, "~>>", 3 ); }

        |   '!='   >ts %{ MTOKEN( YL_TOKEN_NOTEQUAL, sloc, "!=", 2 ); }
        |   '<='   >ts %{ MTOKEN( YL_TOKEN_LESSEQUAL, sloc, "<=", 2 ); }
        |   '=='   >ts %{ MTOKEN( YL_TOKEN_EQUAL, sloc, "==", 2 ); }
        |   '>='   >ts %{ MTOKEN( YL_TOKEN_GREATEREQUAL, sloc, ">=", 2 ); }
        |   '!in'  >ts %{ MTOKEN( YL_TOKEN_NOTIN, sloc, "!in", 3 ); }
        |   '!is'  >ts %{ MTOKEN( YL_TOKEN_NOTIS, sloc, "!is", 3 ); }

        |   '%='   >ts %{ MTOKEN( YL_TOKEN_MODASSIGN, sloc, "%=", 2 ); }
        |   '&='   >ts %{ MTOKEN( YL_TOKEN_BITANDASSIGN, sloc, "&=", 2 ); }
        |   '*='   >ts %{ MTOKEN( YL_TOKEN_MULASSIGN, sloc, "*=", 2 ); }
        |   '+='   >ts %{ MTOKEN( YL_TOKEN_ADDASSIGN, sloc, "+=", 2 ); }
        |   '-='   >ts %{ MTOKEN( YL_TOKEN_SUBASSIGN, sloc, "-=", 2 ); }
        |   '/='   >ts %{ MTOKEN( YL_TOKEN_DIVASSIGN, sloc, "/=", 2 ); }
        |   '^='   >ts %{ MTOKEN( YL_TOKEN_BITXORASSIGN, sloc, "^=", 2 ); }
        |   '|='   >ts %{ MTOKEN( YL_TOKEN_BITORASSIGN, sloc, "|=", 2 ); }
        |   '~='   >ts %{ MTOKEN( YL_TOKEN_INTDIVASSIGN, sloc, "~=", 2 ); }
        |   '<<='  >ts %{ MTOKEN( YL_TOKEN_LSHIFTASSIGN, sloc, "<<=", 3 ); }
        |   '>>='  >ts %{ MTOKEN( YL_TOKEN_LRSHIFTASSIGN, sloc, ">>=", 3 ); }
        |   '~>>=' >ts %{ MTOKEN( YL_TOKEN_ARSHIFTASSIGN, sloc, "~>>=", 4 ); }

        |   '&&'   >ts %{ MTOKEN( YL_TOKEN_LOGICAND, sloc, "&&", 2 ); }
        |   '^^'   >ts %{ MTOKEN( YL_TOKEN_LOGICXOR, sloc, "^^", 2 ); }
        |   '||'   >ts %{ MTOKEN( YL_TOKEN_LOGICOR, sloc, "||", 2 ); }

        |   '::'   >ts %{ MTOKEN( YL_TOKEN_EACHKEY, sloc, "::", 2 ); }
        |   '...'  >ts %{ MTOKEN( YL_TOKEN_ELLIPSIS, sloc, "...", 3 ); }

    )**

    @/neof

    $!nchar $!{ fgoto main; };

}%%

%% write data;

template < typename ... arguments_t >
yl_token* yl_parser::make_token( arguments_t ... arguments )
{
    void* p = nullptr;
    if ( recycle_tokens.size() )
    {
        p = recycle_tokens.back();
        recycle_tokens.pop_back();
    }
    else
    {
        p = malloc( sizeof( yl_token ) );
    }
    return new ( p ) yl_token( arguments ... );
}

struct yl_keywords
    :   public std::unordered_map< symkey, yl_token_kind >
{

    yl_keywords()
    {
        emplace( symkey( "break" ),     YL_KEYWORD_BREAK );
        emplace( symkey( "case" ),      YL_KEYWORD_CASE );
        emplace( symkey( "catch" ),     YL_KEYWORD_CATCH );
        emplace( symkey( "continue" ),  YL_KEYWORD_CONTINUE );
        emplace( symkey( "def" ),       YL_KEYWORD_DEF );
        emplace( symkey( "default" ),   YL_KEYWORD_DEFAULT );
        emplace( symkey( "delete" ),    YL_KEYWORD_DELETE );
        emplace( symkey( "do" ),        YL_KEYWORD_DO );
        emplace( symkey( "else" ),      YL_KEYWORD_ELSE );
        emplace( symkey( "false" ),     YL_KEYWORD_FALSE );
        emplace( symkey( "finally" ),   YL_KEYWORD_FINALLY );
        emplace( symkey( "for" ),       YL_KEYWORD_FOR );
        emplace( symkey( "if" ),        YL_KEYWORD_IF );
        emplace( symkey( "new" ),       YL_KEYWORD_NEW );
        emplace( symkey( "null" ),      YL_KEYWORD_NULL );
        emplace( symkey( "return" ),    YL_KEYWORD_RETURN );
        emplace( symkey( "superof" ),   YL_KEYWORD_SUPEROF );
        emplace( symkey( "switch" ),    YL_KEYWORD_SWITCH );
        emplace( symkey( "throw" ),     YL_KEYWORD_THROW );
        emplace( symkey( "true" ),      YL_KEYWORD_TRUE );
        emplace( symkey( "try" ),       YL_KEYWORD_TRY );
        emplace( symkey( "using" ),     YL_KEYWORD_USING );
        emplace( symkey( "var" ),       YL_KEYWORD_VAR );
        emplace( symkey( "while" ),     YL_KEYWORD_WHILE );
        emplace( symkey( "yield" ),     YL_KEYWORD_YIELD );
        emplace( symkey( "in" ),        YL_TOKEN_IN );
        emplace( symkey( "is" ),        YL_TOKEN_IS );
    }

};

yl_token* yl_parser::make_identifier( int sloc, region_buffer* data )
{
    static yl_keywords keywords;

    size_t length = data->size();
    const char* identifier = (const char*)data->get();
    symkey key( identifier, length );

    // Check for keyword.
    auto i = keywords.find( key );
    if ( i != keywords.end() )
    {
        return make_token( i->second, sloc, i->first.c_str(), i->first.size() );
    }

    // Check for existing identifier.
    auto j = identifiers.find( key );
    if ( j != identifiers.end() )
    {
        return make_token( YL_TOKEN_IDENTIFIER, sloc, j->c_str(), j->size() );
    }

    // New identifier.
    data->append( '\0' );
    identifier = (const char*)data->tearoff();
    identifiers.emplace( key.hash(), identifier, length );
    return make_token( YL_TOKEN_IDENTIFIER, sloc, identifier, length );

}

static bool encode_utf8( region_buffer* data, uint32_t cp )
{
    if ( cp <= 0x7F )
    {
        data->append( (uint8_t)cp );
        return true;
    }
    else if ( cp <= 0x7FF )
    {
        data->append( (uint8_t)( 0xC0 | ( cp >> 6 & 0x1F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else if ( cp <= 0xFFFF )
    {
        data->append( (uint8_t)( 0xE0 | ( cp >> 12 & 0x0F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 6 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else if ( cp <= 0x10FFFF )
    {
        data->append( (uint8_t)( 0xF0 | ( cp >> 18 & 0x07 ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 12 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp >> 6 & 0x3F ) ) );
        data->append( (uint8_t)( 0x80 | ( cp & 0x3F ) ) );
        return true;
    }
    else
    {
        return false;
    }
}

bool yl_parser::parse( const char* path )
{
    region_scope rscope( root->alloc );
    region_buffer data( root->alloc );

    // Push function scope.
    scopes.push_back( root->function->scope );

    // Open file.
    diagnostics->set_filename( path );
    FILE* file = fopen( path, "r" );
    if ( ! file )
    {
        diagnostics->error( 0, "unable to open file" );
        return false;
    }

    // Name of root function.
    root->function->funcname = root->alloc.strdup( path );

    // File state.
    int offset = 0;
    unsigned char* buffer = (unsigned char*)malloc( BUFFER_SIZE );

    // Token state.
    int sloc = -1;
    uint32_t temp = 0;

    // Parser state.
    void* parser = XecParseAlloc( malloc );

    // Initialize state machine.
    int cs;

    %% write init;

    // Perform lexing.
#define TOKEN( token ) \
    { \
        XecParse( parser, token->kind, token, this ); \
        if ( diagnostics->error_count() >= ERROR_LIMIT ) \
            goto error; \
    }
#define MTOKEN( ... ) \
    { \
        yl_token* token = make_token( __VA_ARGS__ ); \
        XecParse( parser, token->kind, token, this ); \
        if ( diagnostics->error_count() >= ERROR_LIMIT ) \
            goto error; \
    }

    bool iseof = false;
    while ( ! iseof )
    {
        size_t read = fread( buffer, 1, BUFFER_SIZE, file );
        iseof = read < BUFFER_SIZE;

        if ( iseof && ferror( file ) )
        {
            data.shrink();
            diagnostics->error( offset, "error reading file" );
            goto error;
        }

        const unsigned char* p      = buffer;
        const unsigned char* pe     = buffer + read;
        const unsigned char* eof    = iseof ? pe : nullptr;

        %% write exec;

        offset += read;
    }

    if ( ! ferror( file ) )
    {
        MTOKEN( YL_TOKEN_EOF, offset, "", 0 );
    }

#undef MTOKEN
#undef TOKEN

error:

    // Free parser.
    XecParseFree( parser, free );

    // Close file.
    free( buffer );
    fclose( file );

    // Check if there were errors.
    return diagnostics->error_count() == 0;
}

#pragma unused( lexer_first_final )
#pragma unused( lexer_error )
#pragma unused( lexer_en_main )