- permit carriage-return characters so windows line-terminators (CRLF)   are accepted as whitespace.

require "logstash/namespace"

%%{
  machine logstash_config;

  action mark {
    @tokenstack.push(p)
    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
  }

  action stack_numeric {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "numeric: #{token}"
    #puts "numeric?: #{string[startpos,50]}"
    #puts [startpos, endpos].join(",")
    # TODO(sissel): Don't do 'to_i' here. Type coersion is the job of the
    # plugin and the validator.
    @stack << token.to_i
  }

  action stack_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "string: #{token}"
    @stack << token
  }

  action stack_quoted_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos + 1 ... endpos - 1] # Skip quotations

    # Parse escapes.
    token.gsub(/\\./) { |m| m[1,1] }
    #puts "quotedstring: #{token}"
    @stack << token
  }

  action array_init {
    @array = []
    @stack << :array_init
  }

  action array_push {
    while @stack.last != :array_init
      @array.unshift @stack.pop
    end
    @stack.pop # pop :array_init

    @stack << @array
  }

  action parameter_init {
    # nothing
  }

  action parameter {
    value = @stack.pop
    name = @stack.pop
    #puts "parameter: #{name} => #{value}"
    if value.is_a?(Array)
      @parameters[name] += value
    else
      @parameters[name] << value
    end
  }

  action plugin {
    @components ||= []
    name = @stack.pop
    #@components << { :name => name, :parameters => @parameters }
    @components << { name => @parameters }
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component_init {
    @components = []
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component {
    name = @stack.pop
    @config ||= Hash.new { |h,k| h[k] = [] }
    @config[name] += @components
    #puts "Config component: #{name}"
  }

  #%{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
  comment = "#" (any - [\r\n])* >mark ;
  ws = ([ \t\r\n] | comment)** ;
  #ws = ([ \t\n])** ;

  # TODO(sissel): Support floating point values?
  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
  quoted_string = (
    ( "\"" ( ( (any - [\\"\r\n]) | "\\" any )* ) "\"" )
    | ( "'" ( ( (any - [\\'\r\n]) | "\\" any )* ) "'" )
  ) >mark %stack_quoted_string ;
  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
  string = ( quoted_string | naked_string ) ;

  # TODO(sissel): allow use of this.
  regexp_literal = ( "/" ( ( (any - [\\'\r\n]) | "\\" any )* ) "/" )  ;

  array = ( "[" ws ( string | numeric ) ws ("," ws (string | numeric ) ws)* "]" ) >array_init %array_push;
  # TODO(sissel): Implement hash syntax { key => value, ... }
  # TODO(sissel): hashes should support arrays as values.

  parameter_value = ( numeric | string | array );
  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
  parameters = ( parameter ( ws parameter )** ) >parameter_init ;

  # Statement:
  # component {
  #   plugin_name {
  #     bar => ...
  #     baz => ...
  #   }
  #   ...
  # }

  plugin = (
    (
      naked_string ws "{" ws
        parameters
      ws "}"
    ) | (
      naked_string ws "{" ws "}"
    )
  ) %plugin ;

  component = (
    naked_string ws "{"
      >component_init
      ( ws plugin )**
    ws "}"
  ) %component ;

  config = (ws component? )** ;

  main := config %{ puts "END" }
          $err {
            # Compute line and column of the cursor (p)
            $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
            # TODO(sissel): Note what we were expecting?
          } ;
}%%

class LogStash::Config::Grammar
  attr_accessor :eof
  attr_accessor :config

  def initialize
    # BEGIN RAGEL DATA
    %% write data;
    # END RAGEL DATA

    @tokenstack = Array.new
    @stack = Array.new

    @types = Hash.new { |h,k| h[k] = [] }
    @edges = []
  end

  def parse(string)
    # TODO(sissel): Due to a bug in my parser, we need one trailing whitespace
    # at the end of the string. I'll fix this later.
    string += "\n"

    data = string.unpack("c*")

    # BEGIN RAGEL INIT
    %% write init;
    # END RAGEL INIT

    begin
      # BEGIN RAGEL EXEC
      %% write exec;
      # END RAGEL EXEC
    rescue => e
      # Compute line and column of the cursor (p)
      raise e
    end

    if cs < self.logstash_config_first_final
      $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
      raise "Invalid Configuration. Check syntax of config file."
    end
    return cs
  end # def parse

  def line(str, pos)
    return str[0 .. pos].count("\n") + 1
  end

  def column(str, pos)
    return str[0 .. pos].split("\n").last.length
  end
end # class LogStash::Config::Grammar

#def parse(string)
  #cfgparser = LogStash::Config::Grammar.new
  #result = cfgparser.parse(string)
  #puts "result %s" % result
  #ap cfgparser.config
#end

#parse(File.open(ARGV[0]).read)

require "logstash/namespace"

%%{
  machine logstash_config;

  action mark {
    @tokenstack.push(p)
    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
  }

  action stack_numeric {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "numeric: #{token}"
    #puts "numeric?: #{string[startpos,50]}"
    #puts [startpos, endpos].join(",")
    # TODO(sissel): Don't do 'to_i' here. Type coersion is the job of the
    # plugin and the validator.
    @stack << token.to_i
  }

  action stack_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "string: #{token}"
    @stack << token
  }

  action stack_quoted_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos + 1 ... endpos - 1] # Skip quotations

    # Parse escapes.
    token.gsub(/\\./) { |m| m[1,1] }
    #puts "quotedstring: #{token}"
    @stack << token
  }

  action array_init {
    @array = []
    @stack << :array_init
  }

  action array_push {
    while @stack.last != :array_init
      @array.unshift @stack.pop
    end
    @stack.pop # pop :array_init

    @stack << @array
  }

  action parameter_init {
    # nothing
  }

  action parameter {
    value = @stack.pop
    name = @stack.pop
    #puts "parameter: #{name} => #{value}"
    if value.is_a?(Array)
      @parameters[name] += value
    else
      @parameters[name] << value
    end
  }

  action plugin {
    @components ||= []
    name = @stack.pop
    #@components << { :name => name, :parameters => @parameters }
    @components << { name => @parameters }
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component_init {
    @components = []
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component {
    name = @stack.pop
    @config ||= Hash.new { |h,k| h[k] = [] }
    @config[name] += @components
    #puts "Config component: #{name}"
  }

  #%{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
  comment = "#" (any - [\n])* >mark ;
  ws = ([ \t\n] | comment)** ;
  #ws = ([ \t\n])** ;

  # TODO(sissel): Support floating point values?
  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
  quoted_string = (
    ( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" )
    | ( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" )
  ) >mark %stack_quoted_string ;
  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
  string = ( quoted_string | naked_string ) ;

  # TODO(sissel): allow use of this.
  regexp_literal = ( "/" ( ( (any - [\\'\n]) | "\\" any )* ) "/" )  ;

  array = ( "[" ws ( string | numeric ) ws ("," ws (string | numeric ) ws)* "]" ) >array_init %array_push;
  # TODO(sissel): Implement hash syntax { key => value, ... }
  # TODO(sissel): hashes should support arrays as values.

  parameter_value = ( numeric | string | array );
  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
  parameters = ( parameter ( ws parameter )** ) >parameter_init ;

  # Statement:
  # component {
  #   plugin_name {
  #     bar => ...
  #     baz => ...
  #   }
  #   ...
  # }

  plugin = (
    (
      naked_string ws "{" ws
        parameters
      ws "}"
    ) | (
      naked_string ws "{" ws "}"
    )
  ) %plugin ;

  component = (
    naked_string ws "{"
      >component_init
      ( ws plugin )**
    ws "}"
  ) %component ;

  config = (ws component? )** ;

  main := config %{ puts "END" }
          $err {
            # Compute line and column of the cursor (p)
            $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
            # TODO(sissel): Note what we were expecting?
          } ;
}%%

class LogStash::Config::Grammar
  attr_accessor :eof
  attr_accessor :config

  def initialize
    # BEGIN RAGEL DATA
    %% write data;
    # END RAGEL DATA

    @tokenstack = Array.new
    @stack = Array.new

    @types = Hash.new { |h,k| h[k] = [] }
    @edges = []
  end

  def parse(string)
    # TODO(sissel): Due to a bug in my parser, we need one trailing whitespace
    # at the end of the string. I'll fix this later.
    string += "\n"

    data = string.unpack("c*")

    # BEGIN RAGEL INIT
    %% write init;
    # END RAGEL INIT

    begin
      # BEGIN RAGEL EXEC
      %% write exec;
      # END RAGEL EXEC
    rescue => e
      # Compute line and column of the cursor (p)
      raise e
    end

    if cs < self.logstash_config_first_final
      $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
      raise "Invalid Configuration. Check syntax of config file."
    end
    return cs
  end # def parse

  def line(str, pos)
    return str[0 .. pos].count("\n") + 1
  end

  def column(str, pos)
    return str[0 .. pos].split("\n").last.length
  end
end # class LogStash::Config::Grammar

#def parse(string)
  #cfgparser = LogStash::Config::Grammar.new
  #result = cfgparser.parse(string)
  #puts "result %s" % result
  #ap cfgparser.config
#end

#parse(File.open(ARGV[0]).read)


add space after comma in typized argument lists

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ('<' . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


Escape quotes and backslashes when printing tag values.

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static bool is_game_termination_marker(char *symbol)
{
	return strcmp(symbol, "1-0") == 0 ||
		strcmp(symbol, "0-1") == 0 ||
		strcmp(symbol, "1/2-1/2") == 0;
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;
		if (is_game_termination_marker(t->value.string))
			return true;

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

static void write_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// TODO: handle IO errors
	fprintf(file, "[%s \"" , tag_name);

	for (size_t i = 0; tag_value[i] != '\0'; i++) {
		char c = tag_value[i];
		if (c == '\\' || c == '"')
			putc('\\', file);

		putc(c, file);
	}

	fputs("\"]\n", file);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	// TODO: sorting
	g_hash_table_foreach(pgn->tags, write_tag, (void *)file);

	Game *game = pgn->game->children;
	do {
		if (game->board->turn == WHITE)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		// We use game->parent->board here, as that is the baord in which
		// this move was made.
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	// TODO: write game termination marker - requires storing the result in
	// the Game struct

	putc('\n', file);

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}

#include <assert.h>
#include <ctype.h>
#include <gio/gio.h>
#include <glib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "board.h"
#include "moves.h"
#include "pgn.h"

// TODO: error messages

%%{
	machine pgn_tokenizer;
	write data;
}%%

typedef enum Token_type
{
	// Tokens with different values
	STRING, INTEGER, NAG, SYMBOL,
	// Fixed tokens - always just a single character
	DOT, ASTERISK, L_SQUARE_BRACKET, R_SQUARE_BRACKET,
	L_BRACKET, R_BRACKET, L_ANGLE_BRACKET, R_ANGLE_BRACKET,
} Token_type;

typedef union Token_value {
	char *string;
	uint integer;
} Token_value;

typedef struct Token
{
	Token_type type;
	Token_value value;
} Token;

void print_token(Token *t)
{
	switch (t->type) {
	case STRING: printf("STRING: %s\n", t->value.string); break;
	case SYMBOL: printf("SYMBOL: %s\n", t->value.string); break;
	case NAG: printf("NAG: %s\n", t->value.string); break;
	case INTEGER: printf("INTEGER: %d\n", t->value.integer); break;
	case DOT: puts("DOT"); break;
	case ASTERISK: puts("DOT"); break;
	case L_SQUARE_BRACKET: puts("L_SQUARE_BRACKET"); break;
	case R_SQUARE_BRACKET: puts("R_SQUARE_BRACKET"); break;
	case L_BRACKET: puts("L_BRACKET"); break;
	case R_BRACKET: puts("R_BRACKET"); break;
	case L_ANGLE_BRACKET: puts("L_ANGLE_BRACKET"); break;
	case R_ANGLE_BRACKET: puts("R_ANGLE_BRACKET"); break;
	}
}

bool symbol_is_integer(Token *t)
{
	char *str = t->value.string;
	for (size_t i = 0; str[i] != '\0'; i++)
		if (!isdigit(str[i]))
			return false;

	return true;
}

char *escape_string(char *str, size_t length)
{
	char *out = malloc(length + 1);
	size_t j = 0;
	bool escaping = false;

	for (size_t i = 0; i < length; i++) {
		char c = str[i];
		switch (c) {
		case '\\':
			if (escaping) {
				out[j++] = '\\';
				escaping = false;
			} else {
				escaping = true;
			}

			break;
		case '"':
			// We should never come across an unescaped quote in a string, as
			// the tokenizer should have terminated the string in this case.
			assert(escaping);

			out[j++] = '"';
			escaping = false;
			break;
		default:
			out[j++] = c;
			escaping = false;
			break;
		}
	}

	out[j] = '\0';

	return out;
}

static GArray *tokenize_pgn(char *buf, gsize length)
{
	// The initial size (140) is just a rough estimate of the average number of
	// tokens, based on 4 tokens for each of the 7 required tags, and 4 tokens
	// for each of 30 moves.
	GArray *tokens = g_array_sized_new(FALSE, FALSE, sizeof(Token), 140);

	// Variables that Ragel needs
	char *p = buf, *pe = buf + length;
	char *eof = NULL;
	char *ts, *te;
	int cs, act;
	// act is initialized for scanners, so it must be declared, but in this
	// scanner we don't actually use it. This silences the compiler warning.
	IGNORE(act);

	%%write init;
	%%{
		action add_string {
			// - 2 for quotes
			size_t length = te - ts - 2;
			char *token = escape_string(ts + 1, length);

			Token t = { STRING, { token } };

			g_array_append_val(tokens, t);
		}

		action add_symbol {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { SYMBOL, { token } };

			g_array_append_val(tokens, t);
		}

		action add_nag {
			// + 1 for null terminator
			size_t length = te - ts + 1;
			char *token = malloc(length);
			strncpy(token, ts, length - 1);
			token[length - 1] = '\0';

			Token t = { NAG, { token } };

			g_array_append_val(tokens, t);
		}

		# Token types, as per PGN spec section 7
		# Integers are read as symbols, and we convert them in the second pass

		# We cheat a little bit here.
		# According to the PGN spec, game termination markers are simply symbols.
		# However, according to the definition of the symbol token, symbols
		# cannot contain the '/' character. This seems like a contradiction, so
		# to work around it we allow '/'s in symbols.
		symbol = alnum (alnum | [_+#=:\-/])*;
		string = '"' (('\\' print) | (print - '\\"'))* '"';
		nag = '$' digit+;

		main := |*
			space;
			symbol  => add_symbol;
			string  => add_string;
			nag     => add_nag;
			'.'     => { Token t = { DOT,              { 0 } }; g_array_append_val(tokens, t); };
			'*'     => { Token t = { ASTERISK,         { 0 } }; g_array_append_val(tokens, t); };
			'['     => { Token t = { L_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			']'     => { Token t = { R_SQUARE_BRACKET, { 0 } }; g_array_append_val(tokens, t); };
			'('     => { Token t = { L_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			')'     => { Token t = { R_BRACKET,        { 0 } }; g_array_append_val(tokens, t); };
			'<'     => { Token t = { L_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
			'>'     => { Token t = { R_ANGLE_BRACKET,  { 0 } }; g_array_append_val(tokens, t); };
		*|;

		write exec;
	}%%

	// Integers are a subset of symbols, and unfortunately Ragel scanners
	// attempt to match longer patterns before shorter ones.
	// So we do a second pass to look for symbols that are integers.
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL && symbol_is_integer(t)) {
			int n;
			sscanf(t->value.string, "%d", &n);

			t->type = INTEGER;
			t->value.integer = n;
		}
	}

	return tokens;
}

// It is impossible to parse a move without a reference to a particular board,
// as something like Bd5 could start from any square on that diagonal.
static Move parse_move(Board *board, char *notation)
{
	if (strcmp(notation, "O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(6, rank));
	}
	if (strcmp(notation, "O-O-O") == 0) {
		uint rank = board->turn == WHITE ? 0 : 7;
		return MOVE(SQUARE(4, rank), SQUARE(2, rank));
	}

	char stripped[5]; // max length without 'x#+'s, + 1 for null terminator
	size_t j = 0;
	for (size_t i = 0; notation[i] != '\0'; i++)
		if (notation[i] != 'x' && notation[i] != '#' && notation[i] != '+')
			stripped[j++] = notation[i];
	stripped[j] = '\0';

	size_t i = 0;
	Piece_type type;
	// If it's a pawn move, the target square starts at the beginning of the
	// string
	if (islower(stripped[0])) {
		type = PAWN;

		// An exception: if it's a move like exd5, the target square starts one
		// char further on
		if (islower(stripped[1]))
			i++;
	} else {
		type = PIECE_TYPE(piece_from_char(stripped[0]));
		i++;
	}

	char disambig = 0;
	if (j == 4)
		// If it's this long, there must be a disambiguation char in there
		disambig = stripped[i++];

	uint target_file = CHAR_FILE(stripped[i++]);
	uint target_rank = CHAR_RANK(stripped[i++]);

	if (disambig != 0) {
		uint x = 0, y = 0, dx = 0, dy = 0;
		if (disambig >= 'a' && disambig <= 'g') {
			x = CHAR_FILE(disambig);
			dy = 1;
		} else if (disambig >= '1' && disambig <= '8') {
			y = CHAR_RANK(disambig);
			dx = 1;
		} else {
			return NULL_MOVE;
		}

		for (; x < BOARD_SIZE && y < BOARD_SIZE; x += dx, y += dy) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}

		return NULL_MOVE;
	}

	for (uint x = 0; x < BOARD_SIZE; x++) {
		for (uint y = 0; y < BOARD_SIZE; y++) {
			Piece p = PIECE_AT(board, x, y);
			if (PIECE_TYPE(p) != type || PLAYER(p) != board->turn)
				continue;

			Move m = MOVE(SQUARE(x, y), SQUARE(target_file, target_rank));
			if (legal_move(board, m, true))
				return m;
		}
	}

	return NULL_MOVE;
}

static bool is_game_termination_marker(char *symbol)
{
	return strcmp(symbol, "1-0") == 0 ||
		strcmp(symbol, "0-1") == 0 ||
		strcmp(symbol, "1/2-1/2") == 0;
}

static bool parse_tokens(PGN *pgn, GArray *tokens)
{
	// Start with tags
	pgn->tags = g_hash_table_new(g_str_hash, g_str_equal);

	size_t i = 0;
	while ((&g_array_index(tokens, Token, i))->type == L_SQUARE_BRACKET) {
		Token *tag_name_token = &g_array_index(tokens, Token, ++i);
		if (tag_name_token->type != SYMBOL)
			return false;

		char *tag_name = tag_name_token->value.string;

		if (g_hash_table_contains(pgn->tags, tag_name))
			return false;

		Token *tag_value_token = &g_array_index(tokens, Token, ++i);
		if (tag_value_token->type != STRING)
			return false;

		char *tag_value = tag_value_token->value.string;

		char *tag_name_copy = malloc(strlen(tag_name) + 1);
		strcpy(tag_name_copy, tag_name);
		char *tag_value_copy = malloc(strlen(tag_value) + 1);
		strcpy(tag_value_copy, tag_value);

		g_hash_table_insert(pgn->tags, tag_name_copy, tag_value_copy);

		Token *close_square_bracket_token = &g_array_index(tokens, Token, ++i);
		if (close_square_bracket_token->type != R_SQUARE_BRACKET)
			return false;

		i++;
	}

	// Now the movetext section
	uint half_move_number = 2;

	Game *game = new_game();
	game->board = malloc(sizeof(Board));
	pgn->game = game;
	// TODO: Use value in start board tag if present.
	from_fen(game->board, start_board_fen);

	// TODO: variations, NAG
	while (i < tokens->len) {
		Token *t = &g_array_index(tokens, Token, i++);
		if (t->type == INTEGER) {
			if (t->value.integer != half_move_number / 2)
				return false;

			while ((t = &g_array_index(tokens, Token, i++))->type == DOT)
				;
		}

		if (t->type != SYMBOL)
			return false;
		if (is_game_termination_marker(t->value.string))
			return true;

		Move m;
		if ((m = parse_move(game->board, t->value.string)) == NULL_MOVE)
			return false;

		Board *new_board = malloc(sizeof *new_board);
		copy_board(new_board, game->board);
		perform_move(new_board, m);
		add_child(game, m, new_board);
		game = first_child(game);

		half_move_number++;
	}

	return true;
}

void free_tokens(GArray *tokens)
{
	for (size_t i = 0; i < tokens->len; i++) {
		Token *t = &g_array_index(tokens, Token, i);
		if (t->type == SYMBOL || t->type == STRING || t->type == NAG)
			free(t->value.string);
	}

	g_array_free(tokens, TRUE);
}

bool read_pgn(PGN *pgn, const char *input_filename, GError **error)
{
	assert(*error == NULL);

	bool ret = true;
	char *buf;
	gsize length;

	GFile *file = g_file_new_for_path(input_filename);
	if (!g_file_load_contents(file, NULL, &buf, &length, NULL, error)) {
		ret = false;
		goto cleanup;
	}

	GArray *tokens = tokenize_pgn(buf, length);
	ret = parse_tokens(pgn, tokens);
	free_tokens(tokens);

cleanup:
	g_free(buf);
	g_object_unref(file);

	return ret;
}

static void write_tag(gpointer key, gpointer value, gpointer user_data)
{
	FILE *file = (FILE *)user_data;
	char *tag_name = (char *)key;
	char *tag_value = (char *)value;

	// TODO: handle IO errors
	fprintf(file, "[%s \"%s\"]\n", tag_name, tag_value);
}

// TODO: handle IO errors
bool write_pgn(PGN *pgn, FILE *file)
{
	// TODO: sorting
	g_hash_table_foreach(pgn->tags, write_tag, (void *)file);

	Game *game = pgn->game->children;
	do {
		if (game->board->turn == WHITE)
			fprintf(file, game->board->move_number == 1 ?
				"%d." :
				" %d.", game->board->move_number);

		char move_str[MAX_NOTATION_LENGTH];
		// We use game->parent->board here, as that is the baord in which
		// this move was made.
		move_notation(game->parent->board, game->move, move_str);
		fprintf(file, " %s", move_str);

		game = first_child(game);
	} while (game != NULL);

	// TODO: write game termination marker - requires storing the result in
	// the Game struct

	putc('\n', file);

	return true;
}

void free_pgn(PGN *pgn)
{
	g_hash_table_destroy(pgn->tags);
	free_game(pgn->game);
}


Finally fixed the ambiguity in the regex-based scan of group_options.

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  hex_or_not            = (xdigit|[^0-9a-fA-F}]); # note closing curly at end

  wide_hex_seq_invalid  = 'x' . '{' . hex_or_not{1,9};
  wide_hex_seq_empty    = 'x' . '{' . (space+)? . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-');

  meta_sequence         = 'M-' . (backslash . control_sequence)?;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . [\-mix];

  group_ref             = [gk];
  group_name            = (alnum . (alnum+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");

  group_name_ref        = group_ref . (('<' . group_name . group_level? '>') |
                                       ("'" . group_name . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise InvalidSequenceError.new('sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { group_depth += 1; in_group = true }
  action group_closed { group_depth -= 1; in_group = group_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :close, *text(data, ts, te))

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-]' { # special case, emits two tokens
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :member, copy(data, ts..te-2), ts, te)
      emit(set_type, :close,  copy(data, ts+1..te-1), ts, te)

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '^' {
      text = text(data, ts, te).first
      if @tokens.last[1] == :open
        emit(set_type, :negate, text, ts, te)
      else
        emit(set_type, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum {
      emit(set_type, :range, *text(data, ts, te))
    };

    '&&' {
      emit(set_type, :intersection, *text(data, ts, te))
    };

    '\\' {
      fcall set_escape_sequence;
    };

    '[' >(open_bracket, 1) {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @eof(premature_end_error) {
      text = text(data, ts, te).first

      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = "non#{class_name[1..-1]}"
      end

      token_sym = "class_#{class_name}".to_sym
      emit(set_type, token_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :equivalent, *text(data, ts, te))
    };

    # exclude the closing bracket as a cleaner workaround for dealing with the
    # ambiguity caused upon exit from the unicode properties machine
    meta_char -- ']' {
      emit(set_type, :member, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      emit(set_type, :member, *text(data, ts, te))
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' > (escaped_set_alpha, 2) {
      emit(set_type, :backspace, *text(data, ts, te, 1))
      fret;
    };

    char_type {
      case text = text(data, ts, te, 1).first
      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      emit(set_type, :range_hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence {
      emit(set_type, :member_hex, *text(data, ts, te, 1))
      fret;
    };

    meta_char | [\\\]\-\,] {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };

    property_char > (escaped_set_alpha, 3) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; emit(:escape, :space,          text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex_wide, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    (wide_hex_seq_invalid | wide_hex_seq_empty) {
      raise InvalidSequenceError.new("wide hex sequence")
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("control sequence")
        end
      else
        raise PrematureEndError.new("control sequence")
      end
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("meta sequence")
        end
      else
        raise PrematureEndError.new("meta sequence")
      end
      fret;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext main;
      fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      emit(:meta, :alternation, *text(data, ts, te))
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in anchor at #{text} (char #{ts})")
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = text(data, ts, te).first
      when '\\d'; emit(:type, :digit,      text, ts, te)
      when '\\D'; emit(:type, :nondigit,   text, ts, te)
      when '\\h'; emit(:type, :hex,        text, ts, te)
      when '\\H'; emit(:type, :nonhex,     text, ts, te)
      when '\\s'; emit(:type, :space,      text, ts, te)
      when '\\S'; emit(:type, :nonspace,   text, ts, te)
      when '\\w'; emit(:type, :word,       text, ts, te)
      when '\\W'; emit(:type, :nonword,    text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in type at #{text} (char #{ts})")
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      p = scan_options(p, data, ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)

      when /^\(\?<(\w*)>/
        empty_name_error(:group, 'named group (ab)') if $1.empty?

        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'(\w*)'/
        empty_name_error(:group, 'named group (sq)') if $1.empty?

        emit(:group, :named_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown subexpression group format '#{text}'")
      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      emit(:group, :close, *text(data, ts, te))
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])<>/ # angle brackets
        empty_backref_error("ref/call (ab)")

      when /^\\([gk])''/ # single quotes
        empty_backref_error("ref/call (sq)")

      when /^\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab,  text, ts, te)
        else
          emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /^\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq,  text, ts, te)
        else
          emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /^\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /^\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_rel_call_sq,  text, ts, te)
        end

      when /^\\k<[^\d-](\w+)?[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_nest_ref_ab,  text, ts, te)

      when /^\\k'[^\d-](\w+)?[+\-]\d+'/ # single-quotes
        emit(:backref, :name_nest_ref_sq,  text, ts, te)

      when /^\\([gk])<\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_nest_ref_ab,  text, ts, te)

      when /^\\([gk])'\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_nest_ref_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown backreference format '#{text}'")
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from scanner.rl

module Regexp::Scanner
  %% write data;

  # General scanner error (catch all)
  class ScannerError < StandardError
    def initialize(what)
      super what
    end
  end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group, group_depth = false, 0
    in_set,   set_depth, set_type   = false, 0, :set

    %% write init;
    %% write exec;

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{in_group}:#{group_depth}]") if in_group
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{in_set}:#{set_depth}]") if in_set

    # when the entire expression is a literal run
    emit_literal if @literal

    @tokens
  end

  private

  def self.scan_options(p, data, ts, te)
    text = text(data, ts, te).first

    in_options = true
    in_count = 0

    while in_options
      if data[te + in_count]
        c = data[te + in_count].chr

        if c =~ /[-mix]/
          text << c
          p += 1
          in_count += 1
        else
          in_options = false
        end
      else
        raise PrematureEndError.new("expression options `#{text}'")
      end
    end

    if data[te + in_count]
      c = data[te + in_count].chr

      if c == ':'
        text << c
        p += 1
        in_count += 1
        emit(:group, :options, text, ts, te + in_count)
      elsif c == ')'
        emit(:group, :options, text, ts, te + in_count)
      else
        raise ScannerError.new(
          "Unexpected `#{c}' in options sequence, ':' or ')' expected")
      end
    else
      raise PrematureEndError.new("expression options `#{text}'")
    end

    p
  end

  # Copy from ts to te from data as text
  def self.copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def self.text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    @literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  # Emits an array with the details of the scanned pattern
  def self.emit(type, token, text, ts, te)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if @literal

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def self.validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    else
      error = ValidationError.new('unknown')
    end

    # TODO: configuration option to treat scanner level validation
    # errors as warnings or ignore them
    if false # @@config.validation_warn
      $stderr.puts error.to_s # unless @@config.validation_ignore
    else
      raise error # unless @@config.validation_ignore
    end
  end

  # Used for references with an empty name or number
  def self.empty_backref_error(type, what)
    validation_error(:backref, what, 'ref ID is empty')
  end

  # Used for named expressions with an empty name
  def self.empty_name_error(type, what)
    validation_error(type, what, 'name is empty')
  end

end # module Regexp::Scanner

%%{
  machine re_scanner;
  include re_property "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrstv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];
  wide_hex_sequence     = 'x' . '{' . xdigit{1,8} . '}';

  hex_or_not            = (xdigit|[^0-9a-fA-F}]); # note closing curly at end

  wide_hex_seq_invalid  = 'x' . '{' . hex_or_not{1,9};
  wide_hex_seq_empty    = 'x' . '{' . (space+)? . '}';

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . (xdigit{4} . space?)+'}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-');

  meta_sequence         = 'M-' . (backslash . control_sequence)?;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-'? . ([mix]{1,3})?;

  group_ref             = [gk];
  group_name            = (alnum . (alnum+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_named           = ('?<' . group_name . '>') | ("?'" . group_name . "'");

  group_name_ref        = group_ref . (('<' . group_name . group_level? '>') |
                                       ("'" . group_name . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char)+;
  ascii_nonprint        = (0x01..0x1f | 0x7f)+;

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf)+;
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf)+;
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)+;
  utf8_byte_sequence    = utf8_2_byte | utf8_3_byte | utf8_4_byte;

  non_literal_escape    = char_type | anchor_char | escaped_ascii |
                          group_ref | [xucCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise InvalidSequenceError.new('sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { group_depth += 1; in_group = true }
  action group_closed { group_depth -= 1; in_group = group_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :close, *text(data, ts, te))

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-]' { # special case, emits two tokens
      set_type  = set_depth > 1 ? :subset : :set
      set_depth -= 1; in_set = set_depth > 0 ? true : false

      emit(set_type, :member, copy(data, ts..te-2), ts, te)
      emit(set_type, :close,  copy(data, ts+1..te-1), ts, te)

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '^' {
      text = text(data, ts, te).first
      if @tokens.last[1] == :open
        emit(set_type, :negate, text, ts, te)
      else
        emit(set_type, :member, text, ts, te)
      end
    };

    alnum . '-' . alnum {
      emit(set_type, :range, *text(data, ts, te))
    };

    '&&' {
      emit(set_type, :intersection, *text(data, ts, te))
    };

    '\\' {
      fcall set_escape_sequence;
    };

    '[' >(open_bracket, 1) {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @eof(premature_end_error) {
      text = text(data, ts, te).first

      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = "non#{class_name[1..-1]}"
      end

      token_sym = "class_#{class_name}".to_sym
      emit(set_type, token_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
      emit(set_type, :equivalent, *text(data, ts, te))
    };

    # exclude the closing bracket as a cleaner workaround for dealing with the
    # ambiguity caused upon exit from the unicode properties machine
    meta_char -- ']' {
      emit(set_type, :member, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      emit(set_type, :member, *text(data, ts, te))
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    'b' > (escaped_set_alpha, 2) {
      emit(set_type, :backspace, *text(data, ts, te, 1))
      fret;
    };

    char_type {
      case text = text(data, ts, te, 1).first
      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
      end
      fret;
    };

    hex_sequence . '-\\' . hex_sequence {
      emit(set_type, :range_hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence {
      emit(set_type, :member_hex, *text(data, ts, te, 1))
      fret;
    };

    meta_char | [\\\]\-\,] {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };

    property_char > (escaped_set_alpha, 3) {
      fhold;
      fnext character_set;
      fcall unicode_property;
      fret;
    };

    # special case exclusion of escaped dash, could be cleaner.
    (ascii_print - char_type -- [\-}]) > (escaped_set_alpha, 1) |
    ascii_nonprint            |
    utf8_2_byte               |
    utf8_3_byte               |
    utf8_4_byte               {
      emit(set_type, :escape, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\s'; emit(:escape, :space,          text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    wide_hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex_wide, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    (wide_hex_seq_invalid | wide_hex_seq_empty) {
      raise InvalidSequenceError.new("wide hex sequence")
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :control, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("control sequence")
        end
      else
        raise PrematureEndError.new("control sequence")
      end
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      if data[te]
        c = data[te].chr
        if c =~ /[\x00-\x7F]/
          emit(:escape, :meta_sequence, copy(data, ts-1..te), ts-1, te+1)
          p += 1
        else
          raise InvalidSequenceError.new("meta sequence")
        end
      else
        raise PrematureEndError.new("meta sequence")
      end
      fret;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext main;
      fcall unicode_property; fret;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      emit(:meta, :alternation, *text(data, ts, te))
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in anchor at #{text} (char #{ts})")
      end
    };

    # Character types
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    space, non-space
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    backslash . char_type > (backslashed, 2) {
      case text = text(data, ts, te).first
      when '\\d'; emit(:type, :digit,      text, ts, te)
      when '\\D'; emit(:type, :nondigit,   text, ts, te)
      when '\\h'; emit(:type, :hex,        text, ts, te)
      when '\\H'; emit(:type, :nonhex,     text, ts, te)
      when '\\s'; emit(:type, :space,      text, ts, te)
      when '\\S'; emit(:type, :nonspace,   text, ts, te)
      when '\\w'; emit(:type, :word,       text, ts, te)
      when '\\W'; emit(:type, :nonword,    text, ts, te)
      else
        raise ScannerError.new(
          "Unexpected character in type at #{text} (char #{ts})")
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open {
      set_depth += 1; in_set = true
      set_type  = set_depth > 1 ? :subset : :set

      emit(set_type, :open, *text(data, ts, te))
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      # special handling to resolve ambiguity with passive groups
      if data[te]
        c = data[te].chr
        if c == ':' # include the ':'
          emit(:group, :options, copy(data, ts..te), ts, te+1)
          p += 1
        elsif c == ')' # just options by themselves
          emit(:group, :options, *text(data, ts, te))
        else
          raise ScannerError.new(
            "Unexpected '#{c}' in options sequence, ':' or ')' expected")
        end
      else
        raise PrematureEndError.new("options") unless data[te]
      end
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)

      when /^\(\?<(\w*)>/
        empty_name_error(:group, 'named group (ab)') if $1.empty?

        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'(\w*)'/
        empty_name_error(:group, 'named group (sq)') if $1.empty?

        emit(:group, :named_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown subexpression group format '#{text}'")
      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      emit(:group, :close, *text(data, ts, te))
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])<>/ # angle brackets
        empty_backref_error("ref/call (ab)")

      when /^\\([gk])''/ # single quotes
        empty_backref_error("ref/call (sq)")

      when /^\\([gk])<[^\d-](\w+)?>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab,  text, ts, te)
        else
          emit(:backref, :name_call_ab,  text, ts, te)
        end

      when /^\\([gk])'[^\d-](\w+)?'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq,  text, ts, te)
        else
          emit(:backref, :name_call_sq,  text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_call_ab,  text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_call_sq,  text, ts, te)
        end

      when /^\\([gk])<-\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab,  text, ts, te)
        else
          emit(:backref, :number_rel_call_ab,  text, ts, te)
        end

      when /^\\([gk])'-\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq,  text, ts, te)
        else
          emit(:backref, :number_rel_call_sq,  text, ts, te)
        end

      when /^\\k<[^\d-](\w+)?[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_nest_ref_ab,  text, ts, te)

      when /^\\k'[^\d-](\w+)?[+\-]\d+'/ # single-quotes
        emit(:backref, :name_nest_ref_sq,  text, ts, te)

      when /^\\([gk])<\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_nest_ref_ab,  text, ts, te)

      when /^\\([gk])'\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_nest_ref_sq,  text, ts, te)

      else
        raise ScannerError.new(
          "Unknown backreference format '#{text}'")
      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    ascii_print+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from scanner.rl

module Regexp::Scanner
  %% write data;

  # General scanner error (catch all)
  class ScannerError < StandardError
    def initialize(what)
      super what
    end
  end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input, &block)
    top, stack = 0, []

    input = input.source if input.is_a?(Regexp)
    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    @tokens = []
    @block  = block_given? ? block : nil

    in_group, group_depth = false, 0
    in_set,   set_depth, set_type   = false, 0, :set

    %% write init;
    %% write exec;

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{in_group}:#{group_depth}]") if in_group
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{in_set}:#{set_depth}]") if in_set

    # when the entire expression is a literal run
    emit_literal if @literal

    @tokens
  end

  private

  # Copy from ts to te from data as text
  def self.copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def self.text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def self.append_literal(data, ts, te)
    @literal ||= []
    @literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def self.emit_literal
    ts, te = @literal.first[1], @literal.last[2]
    text = @literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    @literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  # Emits an array with the details of the scanned pattern
  def self.emit(type, token, text, ts, te)
    emit_literal if @literal

    if @block
      @block.call type, token, text, ts, te
    end

    @tokens << [type, token, text, ts, te]
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def self.validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    else
      error = ValidationError.new('unknown')
    end

    # TODO: configuration option to treat scanner level validation
    # errors as warnings or ignore them
    if false # @@config.validation_warn
      $stderr.puts error.to_s # unless @@config.validation_ignore
    else
      raise error # unless @@config.validation_ignore
    end
  end

  # Used for references with an empty name or number
  def self.empty_backref_error(type, what)
    validation_error(:backref, what, 'ref ID is empty')
  end

  # Used for named expressions with an empty name
  def self.empty_name_error(type, what)
    validation_error(type, what, 'name is empty')
  end

end # module Regexp::Scanner