fixed line_nr having wrong type

fixed line_nr having wrong type


Fix segment type when gaps follow first character

– Gap at the next position should cause the current segment type to be set to mixed.

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_mixed_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}


More cleanup of the ragel file

More cleanup of the ragel file

#include "kegogi_tokens.h"
#include "kegogi_parser.h"

#include <stdio.h>

#include <bstring.h>
#include <dbg.h>

%%{
    machine kegogi_lexer;
    defaults = 'defaults';
    send = 'send';
    expect = 'expect';

    comment = '#' [^\n]*;

    action mark { m = fpc; }
    action s1 { s1 = blk2bstr(m, fpc-m); }
    action s2 { s2 = blk2bstr(m, fpc-m); }
    action s3 { s3 = blk2bstr(m, fpc-m); }

    ustring = (alpha (alnum | '-')*) >mark %s1;
    number = (digit+) >mark %s1;
    qstring = '\"' (([^\\\"] | ('\\' any))*) >mark %s1 '\"';
    pattern = '(' (([^\\)] | ('\\' any))*) >mark %s1 ')';
    dict_start = '{';
    dict_end = '}';
    equals = [:=];

    host = ([^/:] - space)+ >mark %s2;
    uri = ('/' (any - space)*) >mark %s1;
    port = digit+ >mark %s3;
    url = ((alpha+ '://')? host (':' port)?)? uri;

    newline = '\n';

    main := |*
         defaults => {
             TokenList_append0(&token_list, TKDEFAULTS);
             s1 = s2 = s3 = NULL;
         };
         send => {
             TokenList_append0(&token_list, TKSEND);
             s1 = s2 = s3 = NULL;
         };
         expect => {
             TokenList_append0(&token_list, TKEXPECT);
             s1 = s2 = s3 = NULL;
         };
         (ustring|qstring) => {
             TokenList_append1(&token_list, TKSTRING, s1);
             s1 = s2 = s3 = NULL;
         };
         pattern => {
             TokenList_append1(&token_list, TKPATTERN, s1);
             s1 = s2 = s3 = NULL;
         };
         number => {
             TokenList_append1(&token_list, TKNUMBER, s1);
             s1 = s2 = s3 = NULL;
         };
         url => {
             TokenList_append3(&token_list, TKURL, s1, s2, s3);
             s1 = s2 = s3 = NULL;
         };
         newline => {
             TokenList_append0(&token_list, TKNEWLINE);
             s1 = s2 = s3 = NULL;
         };
         dict_start => {
             TokenList_append0(&token_list, TKDICT_START);
             s1 = s2 = s3 = NULL;
         };
         dict_end => {
             TokenList_append0(&token_list, TKDICT_END);
             s1 = s2 = s3 = NULL;
         };
         equals => {
             TokenList_append0(&token_list, TKEQUALS);
             s1 = s2 = s3 = NULL;
         };
         space => { };
         ',' => { };
         comment => { };
    *|;
}%%

%% write data;

TokenList *get_kegogi_tokens(bstring content) {
    TokenList *token_list = TokenList_create(1024);
    check_mem(token_list);

    char *p = bdata(content);
    char *pe = p + blength(content);
    char *m = NULL;
    char *eof = pe;
    int cs = -1;
    int act = -1;
    char *ts = NULL;
    char *te = NULL;

    bstring s1 = NULL;
    bstring s2 = NULL;
    bstring s3 = NULL;

    %% write init;
    %% write exec;

    return token_list;

error:
    TokenList_destroy(token_list);
    return NULL;
}

#include "kegogi_tokens.h"
#include "kegogi_parser.h"

#include <stdio.h>

#include <bstring.h>
#include <dbg.h>

%%{
    machine kegogi_lexer;
    defaults = 'defaults';
    send = 'send';
    expect = 'expect';

    comment = '#' [^\n]*;

    action mark { m = fpc; }
    action s1 { s1 = blk2bstr(m, fpc-m); }
    action s2 { s2 = blk2bstr(m, fpc-m); }
    action s3 { s3 = blk2bstr(m, fpc-m); }

    ustring = (alpha (alnum | '-')*) >mark %s1;
    number = (digit+) >mark %s1;
    qstring = '\"' (([^\\\"] | ('\\' any))*) >mark %s1 '\"';
    pattern = '(' (([^\\)] | ('\\' any))*) >mark %s1 ')';
    dict_start = '{';
    dict_end = '}';
    equals = [:=];

    host = [^/: ]+ >mark %s2;
    uri = ('/' (any - space)*) >mark %s1;
    port = digit+ >mark %s3;
    url = ((alpha+ '://')? host (':' port)?)? uri;

    newline = '\n';

    main := |*
         defaults => {
             TokenList_append0(&token_list, TKDEFAULTS);
             s1 = s2 = s3 = NULL;
         };
         send => {
             TokenList_append0(&token_list, TKSEND);
             s1 = s2 = s3 = NULL;
         };
         expect => {
             TokenList_append0(&token_list, TKEXPECT);
             s1 = s2 = s3 = NULL;
         };
         (ustring|qstring) => {
             TokenList_append1(&token_list, TKSTRING, s1);
             s1 = s2 = s3 = NULL;
         };
         pattern => {
             TokenList_append1(&token_list, TKPATTERN, s1);
             s1 = s2 = s3 = NULL;
         };
         number => {
             TokenList_append1(&token_list, TKNUMBER, s1);
             s1 = s2 = s3 = NULL;
         };
         url => {
             TokenList_append3(&token_list, TKURL, s1, s2, s3);
             s1 = s2 = s3 = NULL;
         };
         newline => {
             TokenList_append0(&token_list, TKNEWLINE);
             s1 = s2 = s3 = NULL;
         };
         dict_start => {
             TokenList_append0(&token_list, TKDICT_START);
             s1 = s2 = s3 = NULL;
         };
         dict_end => {
             TokenList_append0(&token_list, TKDICT_END);
             s1 = s2 = s3 = NULL;
         };
         equals => {
             TokenList_append0(&token_list, TKEQUALS);
             s1 = s2 = s3 = NULL;
         };
         space => { };
         ',' => { };
         comment => { };
    *|;
}%%

%% write data;

TokenList *get_kegogi_tokens(bstring content) {
    TokenList *token_list = TokenList_create(1024);
    check_mem(token_list);

    char *p = bdata(content);
    char *pe = p + blength(content);
    char *m = NULL;
    char *eof = pe;
    int cs = -1;
    int act = -1;
    char *ts = NULL;
    char *te = NULL;

    bstring s1 = NULL;
    bstring s2 = NULL;
    bstring s3 = NULL;

    %% write init;
    %% write exec;

    return token_list;

error:
    TokenList_destroy(token_list);
    return NULL;
}


Scanner cleanup - remove deducible ivars

%%{
  machine re_scanner;
  include re_char_type "char_type.rl";
  include re_property  "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  comment               = ('#' . [^\n]* . '\n');

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrtv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;

  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  conditional           = '(?(';

  group_comment         = '?#' . [^)]* . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_absence         = '?~';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  # try to treat every other group head as options group, like Ruby
  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;

  group_ref             = [gk];
  group_name_char       = (alnum | '_');
  group_name_id         = (group_name_char . (group_name_char+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_name            = ('<' . group_name_id . '>') | ("'" . group_name_id . "'");
  group_lookup          = group_name | group_number;

  group_named           = ('?' . group_name );

  group_name_ref        = group_ref . (('<' . group_name_id . group_level? '>') |
                                       ("'" . group_name_id . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_absence | group_named;

  keep_mark             = 'K';

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char);
  ascii_nonprint        = (0x01..0x1f | 0x7f);

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);

  non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
                          group_ref | keep_mark | [xucCM];

  non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
                          multi_codepoint_char_type | [0-9cCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    validation_error(:sequence, 'sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { self.group_depth = group_depth + 1 }
  action group_closed { self.group_depth = group_depth - 1 }
  action set_opened   { self.set_depth   = set_depth   + 1 }
  action set_closed   { self.set_depth   = set_depth   - 1 }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    set_close > (set_meta, 2) @set_closed {
      emit(:set, :close, *text(data, ts, te))
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-]' @set_closed { # special case, emits two tokens
      emit(:literal, :literal, copy(data, ts..te-2), ts, te - 1)
      emit(:set, :close, copy(data, ts+1..te-1), ts + 1, te)
      if in_set?
        fret;
      else
        fgoto main;
      end
    };

    '-&&' { # special case, emits two tokens
      emit(:literal, :literal, '-', ts, te)
      emit(:set, :intersection, '&&', ts, te)
    };

    '^' {
      text = text(data, ts, te).first
      if tokens.last[1] == :open
        emit(:set, :negate, text, ts, te)
      else
        emit(:literal, :literal, text, ts, te)
      end
    };

    '-' {
      text = text(data, ts, te).first
      # ranges cant start with a subset or intersection/negation/range operator
      if tokens.last[0] == :set
        emit(:literal, :literal, text, ts, te)
      else
        emit(:set, :range, text, ts, te)
      end
    };

    # Unlike ranges, intersections can start or end at set boundaries, whereupon
    # they match nothing: r = /[a&&]/; [r =~ ?a, r =~ ?&] # => [nil, nil]
    '&&' {
      emit(:set, :intersection, *text(data, ts, te))
    };

    backslash {
      fcall set_escape_sequence;
    };

    set_open >(open_bracket, 1) >set_opened {
      emit(:set, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
      text = text(data, ts, te).first

      type = :posixclass
      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = class_name[1..-1]
        type = :nonposixclass
      end

      emit(type, class_name.to_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
      emit(:set, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
      emit(:set, :equivalent, *text(data, ts, te))
    };

    meta_char > (set_meta, 1) {
      emit(:literal, :literal, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      char, *rest = *text(data, ts, te)
      char.force_encoding('utf-8') if char.respond_to?(:force_encoding)
      emit(:literal, :literal, char, *rest)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    non_set_escape > (escaped_set_alpha, 2) {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };

    any > (escaped_set_alpha, 1) {
      fhold;
      fnext character_set;
      fcall escape_sequence;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\b'; emit(:escape, :backspace,      text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :control)
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :meta_sequence)
      fret;
    };

    char_type_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall char_type;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set? ? fentry(character_set) : fentry(main));
      fcall unicode_property;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # conditional expressions scanner
  # --------------------------------------------------------------------------
  conditional_expression := |*
    group_lookup . ')' {
      text = text(data, ts, te-1).first
      emit(:conditional, :condition, text, ts, te-1)
      emit(:conditional, :condition_close, ')', te-1, te)
    };

    any {
      fhold;
      fcall main;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      if conditional_stack.last == group_depth
        emit(:conditional, :separator, *text(data, ts, te))
      else
        emit(:meta, :alternation, *text(data, ts, te))
      end
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . keep_mark > (backslashed, 4) {
      emit(:keep, :mark, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open >set_opened {
      emit(:set, :open, *text(data, ts, te))
      fcall character_set;
    };

    # Conditional expression
    #   (?(condition)Y|N)   conditional expression
    # ------------------------------------------------------------------------
    conditional {
      text = text(data, ts, te).first

      conditional_stack << group_depth

      emit(:conditional, :open, text[0..-2], ts, te-1)
      emit(:conditional, :condition_open, '(', te-1, te)
      fcall conditional_expression;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imxdau-imx)         option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #                         d: default class rules (1.9 compatible)
    #                         a: ASCII class rules (\s, \w, etc.)
    #                         u: Unicode class rules (\s, \w, etc.)
    #
    #   (?imxdau-imx:subexp)  option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      text = text(data, ts, te).first
      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
        raise InvalidGroupOption.new($1 || "-#{$2}", text)
      end
      emit_options(text, ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?~subexp)          absence group, matches anything that is not subexp
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)
      when '(?~';  emit(:group, :absence,      text, ts, te)

      when /^\(\?(?:<>|'')/
        validation_error(:group, 'named group', 'name is empty')

      when /^\(\?<\w*>/
        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'\w*'/
        emit(:group, :named_sq,  text, ts, te)

      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      if conditional_stack.last == group_depth + 1
        conditional_stack.pop
        emit(:conditional, :close, *text(data, ts, te))
      else
        if spacing_stack.length > 1 &&
           spacing_stack.last[:depth] == group_depth + 1
          spacing_stack.pop
          self.free_spacing = spacing_stack.last[:free_spacing]
        end

        emit(:group, :close, *text(data, ts, te))
      end
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])(<>|'')/ # angle brackets
        validation_error(:backref, 'ref/call', 'ref ID is empty')

      when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab, text, ts, te)
        else
          emit(:backref, :name_call_ab, text, ts, te)
        end

      when /^\\([gk])'[^\d+-]\w*'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq, text, ts, te)
        else
          emit(:backref, :name_call_sq, text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab, text, ts, te)
        else
          emit(:backref, :number_call_ab, text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq, text, ts, te)
        else
          emit(:backref, :number_call_sq, text, ts, te)
        end

      when /^\\(?:g<\+|g<-|(k)<-)\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab, text, ts, te)
        else
          emit(:backref, :number_rel_call_ab, text, ts, te)
        end

      when /^\\(?:g'\+|g'-|(k)'-)\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq, text, ts, te)
        else
          emit(:backref, :number_rel_call_sq, text, ts, te)
        end

      when /^\\k<[^\d+\-]\w*[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_recursion_ref_ab, text, ts, te)

      when /^\\k'[^\d+\-]\w*[+\-]\d+'/ # single-quotes
        emit(:backref, :name_recursion_ref_sq, text, ts, te)

      when /^\\([gk])<[+\-]?\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_recursion_ref_ab, text, ts, te)

      when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_recursion_ref_sq, text, ts, te)

      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    comment {
      if free_spacing
        emit(:free_space, :comment, *text(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    space+ {
      if free_spacing
        emit(:free_space, :whitespace, *text(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    (ascii_print -- space)+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from lib/regexp_parser/scanner/scanner.rl

class Regexp::Scanner
  # General scanner error (catch all)
  class ScannerError < StandardError; end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid groupOption. Used for inline options.
  class InvalidGroupOption < ValidationError
    def initialize(option, text)
      super "Invalid group option #{option} in #{text}"
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input_object, &block)
    new.scan(input_object, &block)
  end

  def scan(input_object, &block)
    self.literal = nil
    stack = []

    if input_object.is_a?(Regexp)
      input = input_object.source
      self.free_spacing = (input_object.options & Regexp::EXTENDED != 0)
    else
      input = input_object
      self.free_spacing = false
    end
    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]

    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    self.tokens = []
    self.block  = block_given? ? block : nil

    self.set_depth = 0
    self.group_depth = 0
    self.conditional_stack = []

    %% write data;
    %% write init;
    %% write exec;

    # to avoid "warning: assigned but unused variable - testEof"
    testEof = testEof

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{group_depth}]") if in_group?
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{set_depth}]") if in_set?

    # when the entire expression is a literal run
    emit_literal if literal

    tokens
  end

  # lazy-load property maps when first needed
  require 'yaml'
  PROP_MAPS_DIR = File.expand_path('../scanner/properties', __FILE__)

  def self.short_prop_map
    @short_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/short.yml")
  end

  def self.long_prop_map
    @long_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/long.yml")
  end

  # Emits an array with the details of the scanned pattern
  def emit(type, token, text, ts, te)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if literal

    if block
      block.call type, token, text, ts, te
    end

    tokens << [type, token, text, ts, te]
  end

  private

  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                :group_depth, :set_depth, :conditional_stack

  def in_group?
    group_depth > 0
  end

  def in_set?
    set_depth > 0
  end

  # Copy from ts to te from data as text
  def copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def append_literal(data, ts, te)
    self.literal = literal || []
    literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def emit_literal
    ts, te = literal.first[1], literal.last[2]
    text = literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    self.literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  def emit_options(text, ts, te)
    token = nil

    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
    positive, negative, group_local = $1, $2, $3

    if positive.include?('x')
      self.free_spacing = true
    end

    # If the x appears in both, treat it like ruby does, the second cancels
    # the first.
    if negative && negative.include?('x')
      self.free_spacing = false
    end

    if group_local
      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
      token = :options
    else
      # switch for parent group level
      spacing_stack.last[:free_spacing] = free_spacing
      token = :options_switch
    end

    emit(:group, token, text, ts, te)
  end

  def emit_meta_control_sequence(data, ts, te, token)
    if data.last < 0x00 || data.last > 0x7F
      validation_error(:sequence, 'escape', token.to_s)
    end
    emit(:escape, token, *text(data, ts, te, 1))
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    end

    raise error # unless @@config.validation_ignore
  end
end # module Regexp::Scanner

%%{
  machine re_scanner;
  include re_char_type "char_type.rl";
  include re_property  "property.rl";

  dot                   = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  comment               = ('#' . [^\n]* . '\n');

  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  class_posix           = ('[:' . '^'? . class_name_posix . ':]');

  # these are not supported in ruby, and need verification
  collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
  character_equivalent  = '[=' . alpha . '=]';

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];

  escaped_ascii         = [abefnrtv];
  octal_sequence        = [0-7]{1,3};

  hex_sequence          = 'x' . xdigit{1,2};
  hex_sequence_err      = 'x' . [^0-9a-fA-F{];

  codepoint_single      = 'u' . xdigit{4};
  codepoint_list        = 'u{' . xdigit{1,6} . (space . xdigit{1,6})* . '}';
  codepoint_sequence    = codepoint_single | codepoint_list;

  control_sequence      = ('c' | 'C-') . (backslash . 'M-')? . backslash? . any;

  meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_interval;

  conditional           = '(?(';

  group_comment         = '?#' . [^)]* . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';
  group_absence         = '?~';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  # try to treat every other group head as options group, like Ruby
  group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;

  group_ref             = [gk];
  group_name_char       = (alnum | '_');
  group_name_id         = (group_name_char . (group_name_char+)?)?;
  group_number          = '-'? . [1-9] . ([0-9]+)?;
  group_level           = [+\-] . [0-9]+;

  group_name            = ('<' . group_name_id . '>') | ("'" . group_name_id . "'");
  group_lookup          = group_name | group_number;

  group_named           = ('?' . group_name );

  group_name_ref        = group_ref . (('<' . group_name_id . group_level? '>') |
                                       ("'" . group_name_id . group_level? "'"));

  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
                                       ("'" . group_number . group_level? "'"));

  group_type            = group_atomic | group_passive | group_absence | group_named;

  keep_mark             = 'K';

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters that 'break' a literal
  meta_char             = dot | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  ascii_print           = ((0x20..0x7e) - meta_char);
  ascii_nonprint        = (0x01..0x1f | 0x7f);

  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);

  non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
                          group_ref | keep_mark | [xucCM];

  non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
                          multi_codepoint_char_type | [0-9cCM];

  # EOF error, used where it can be detected
  action premature_end_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    raise PrematureEndError.new( text )
  }

  # Invalid sequence error, used from sequences, like escapes and sets
  action invalid_sequence_error {
    text = ts ? copy(data, ts-1..-1) : data.pack('c*')
    validation_error(:sequence, 'sequence', text)
  }

  # group (nesting) and set open/close actions
  action group_opened { self.group_depth = group_depth + 1; in_group = true }
  action group_closed { self.group_depth = group_depth - 1; in_group = group_depth > 0 ? true : false }

  # Character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    set_close > (set_meta, 2) {
      set_depth -= 1
      in_set = set_depth > 0 ? true : false

      emit(:set, :close, *text(data, ts, te))

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-]' { # special case, emits two tokens
      set_depth -= 1
      in_set = set_depth > 0 ? true : false

      emit(:literal, :literal, copy(data, ts..te-2), ts, te-1)
      emit(:set, :close, copy(data, ts+1..te-1), ts+1, te)

      if set_depth == 0
        fgoto main;
      else
        fret;
      end
    };

    '-&&' { # special case, emits two tokens
      emit(:literal, :literal, '-', ts, te)
      emit(:set, :intersection, '&&', ts, te)
    };

    '^' {
      text = text(data, ts, te).first
      if tokens.last[1] == :open
        emit(:set, :negate, text, ts, te)
      else
        emit(:literal, :literal, text, ts, te)
      end
    };

    '-' {
      text = text(data, ts, te).first
      # ranges cant start with a subset or intersection/negation/range operator
      if tokens.last[0] == :set
        emit(:literal, :literal, text, ts, te)
      else
        emit(:set, :range, text, ts, te)
      end
    };

    # Unlike ranges, intersections can start or end at set boundaries, whereupon
    # they match nothing: r = /[a&&]/; [r =~ ?a, r =~ ?&] # => [nil, nil]
    '&&' {
      emit(:set, :intersection, *text(data, ts, te))
    };

    backslash {
      fcall set_escape_sequence;
    };

    set_open >(open_bracket, 1) {
      set_depth += 1

      emit(:set, :open, *text(data, ts, te))
      fcall character_set;
    };

    class_posix >(open_bracket, 1) @eof(premature_end_error) {
      text = text(data, ts, te).first

      type = :posixclass
      class_name = text[2..-3]
      if class_name[0].chr == '^'
        class_name = class_name[1..-1]
        type = :nonposixclass
      end

      emit(type, class_name.to_sym, text, ts, te)
    };

    collating_sequence >(open_bracket, 1) @eof(premature_end_error) {
      emit(:set, :collation, *text(data, ts, te))
    };

    character_equivalent >(open_bracket, 1) @eof(premature_end_error) {
      emit(:set, :equivalent, *text(data, ts, te))
    };

    meta_char > (set_meta, 1) {
      emit(:literal, :literal, *text(data, ts, te))
    };

    any            |
    ascii_nonprint |
    utf8_2_byte    |
    utf8_3_byte    |
    utf8_4_byte    {
      char, *rest = *text(data, ts, te)
      char.force_encoding('utf-8') if char.respond_to?(:force_encoding)
      emit(:literal, :literal, char, *rest)
    };
  *|;

  # set escapes scanner
  # --------------------------------------------------------------------------
  set_escape_sequence := |*
    non_set_escape > (escaped_set_alpha, 2) {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };

    any > (escaped_set_alpha, 1) {
      fhold;
      fnext character_set;
      fcall escape_sequence;
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    [1-9] {
      text = text(data, ts, te, 1).first
      emit(:backref, :number, text, ts-1, te)
      fret;
    };

    octal_sequence {
      emit(:escape, :octal, *text(data, ts, te, 1))
      fret;
    };

    meta_char {
      case text = text(data, ts, te, 1).first
      when '\.';  emit(:escape, :dot,               text, ts-1, te)
      when '\|';  emit(:escape, :alternation,       text, ts-1, te)
      when '\^';  emit(:escape, :bol,               text, ts-1, te)
      when '\$';  emit(:escape, :eol,               text, ts-1, te)
      when '\?';  emit(:escape, :zero_or_one,       text, ts-1, te)
      when '\*';  emit(:escape, :zero_or_more,      text, ts-1, te)
      when '\+';  emit(:escape, :one_or_more,       text, ts-1, te)
      when '\(';  emit(:escape, :group_open,        text, ts-1, te)
      when '\)';  emit(:escape, :group_close,       text, ts-1, te)
      when '\{';  emit(:escape, :interval_open,     text, ts-1, te)
      when '\}';  emit(:escape, :interval_close,    text, ts-1, te)
      when '\[';  emit(:escape, :set_open,          text, ts-1, te)
      when '\]';  emit(:escape, :set_close,         text, ts-1, te)
      when "\\\\";
        emit(:escape, :backslash, text, ts-1, te)
      end
      fret;
    };

    escaped_ascii > (escaped_alpha, 7) {
      # \b is emitted as backspace only when inside a character set, otherwise
      # it is a word boundary anchor. A syntax might "normalize" it if needed.
      case text = text(data, ts, te, 1).first
      when '\a'; emit(:escape, :bell,           text, ts-1, te)
      when '\b'; emit(:escape, :backspace,      text, ts-1, te)
      when '\e'; emit(:escape, :escape,         text, ts-1, te)
      when '\f'; emit(:escape, :form_feed,      text, ts-1, te)
      when '\n'; emit(:escape, :newline,        text, ts-1, te)
      when '\r'; emit(:escape, :carriage,       text, ts-1, te)
      when '\t'; emit(:escape, :tab,            text, ts-1, te)
      when '\v'; emit(:escape, :vertical_tab,   text, ts-1, te)
      end
      fret;
    };

    codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
      text = text(data, ts, te, 1).first
      if text[2].chr == '{'
        emit(:escape, :codepoint_list, text, ts-1, te)
      else
        emit(:escape, :codepoint,      text, ts-1, te)
      end
      fret;
    };

    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
      emit(:escape, :hex, *text(data, ts, te, 1))
      fret;
    };

    hex_sequence_err @invalid_sequence_error {
      fret;
    };

    control_sequence >(escaped_alpha, 4) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :control)
      fret;
    };

    meta_sequence >(backslashed, 3) $eof(premature_end_error) {
      emit_meta_control_sequence(data, ts, te, :meta_sequence)
      fret;
    };

    char_type_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set ? fentry(character_set) : fentry(main));
      fcall char_type;
    };

    property_char > (escaped_alpha, 2) {
      fhold;
      fnext *(in_set ? fentry(character_set) : fentry(main));
      fcall unicode_property;
    };

    (any -- non_literal_escape) > (escaped_alpha, 1)  {
      emit(:escape, :literal, *text(data, ts, te, 1))
      fret;
    };
  *|;

  # conditional expressions scanner
  # --------------------------------------------------------------------------
  conditional_expression := |*
    group_lookup . ')' {
      text = text(data, ts, te-1).first
      emit(:conditional, :condition, text, ts, te-1)
      emit(:conditional, :condition_close, ')', te-1, te)
    };

    any {
      fhold;
      fcall main;
    };
  *|;

  # Main scanner
  # --------------------------------------------------------------------------
  main := |*

    # Meta characters
    # ------------------------------------------------------------------------
    dot {
      emit(:meta, :dot, *text(data, ts, te))
    };

    alternation {
      if in_conditional and conditional_stack.length > 0 and
         conditional_stack.last[1] == group_depth
        emit(:conditional, :separator, *text(data, ts, te))
      else
        emit(:meta, :alternation, *text(data, ts, te))
      end
    };

    # Anchors
    # ------------------------------------------------------------------------
    beginning_of_line {
      emit(:anchor, :bol, *text(data, ts, te))
    };

    end_of_line {
      emit(:anchor, :eol, *text(data, ts, te))
    };

    backslash . keep_mark > (backslashed, 4) {
      emit(:keep, :mark, *text(data, ts, te))
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = text(data, ts, te).first
      when '\\A'; emit(:anchor, :bos,                text, ts, te)
      when '\\z'; emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; emit(:anchor, :eos_ob_eol,         text, ts, te)
      when '\\b'; emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; emit(:anchor, :nonword_boundary,   text, ts, te)
      when '\\G'; emit(:anchor, :match_start,        text, ts, te)
      end
    };

    # Character sets
    # ------------------------------------------------------------------------
    set_open {
      set_depth += 1
      in_set = true

      emit(:set, :open, *text(data, ts, te))
      fcall character_set;
    };

    # Conditional expression
    #   (?(condition)Y|N)   conditional expression
    # ------------------------------------------------------------------------
    conditional {
      text = text(data, ts, te).first

      in_conditional = true unless in_conditional
      conditional_depth += 1
      conditional_stack << [conditional_depth, group_depth]

      emit(:conditional, :open, text[0..-2], ts, te-1)
      emit(:conditional, :condition_open, '(', te-1, te)
      fcall conditional_expression;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    # special case for close, action performed on all transitions to get the
    # correct closing count.
    # ------------------------------------------------------------------------
    group_open . group_comment $group_closed {
      emit(:group, :comment, *text(data, ts, te))
    };

    # Expression options:
    #   (?imxdau-imx)         option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #                         d: default class rules (1.9 compatible)
    #                         a: ASCII class rules (\s, \w, etc.)
    #                         u: Unicode class rules (\s, \w, etc.)
    #
    #   (?imxdau-imx:subexp)  option on/off for subexp
    # ------------------------------------------------------------------------
    group_open . group_options >group_opened {
      text = text(data, ts, te).first
      if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
        raise InvalidGroupOption.new($1 || "-#{$2}", text)
      end
      emit_options(text, ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type >group_opened {
      case text = text(data, ts, te).first
      when '(?=';  emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?~subexp)          absence group, matches anything that is not subexp
    #   (?<name>subexp)     named group
    #   (?'name'subexp)     named group (single quoted version)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type >group_opened {
      case text = text(data, ts, te).first
      when '(?:';  emit(:group, :passive,      text, ts, te)
      when '(?>';  emit(:group, :atomic,       text, ts, te)
      when '(?~';  emit(:group, :absence,      text, ts, te)

      when /^\(\?(?:<>|'')/
        validation_error(:group, 'named group', 'name is empty')

      when /^\(\?<\w*>/
        emit(:group, :named_ab,  text, ts, te)

      when /^\(\?'\w*'/
        emit(:group, :named_sq,  text, ts, te)

      end
    };

    group_open @group_opened {
      text = text(data, ts, te).first
      emit(:group, :capture, text, ts, te)
    };

    group_close @group_closed {
      if in_conditional and conditional_stack.last and
         conditional_stack.last[1] == (group_depth + 1)

        emit(:conditional, :close, *text(data, ts, te))
        conditional_stack.pop

        if conditional_stack.length == 0
          in_conditional = false
        end
      else
        if spacing_stack.length > 1 and
          spacing_stack.last[:depth] == (group_depth + 1)
          spacing_stack.pop

          self.free_spacing = spacing_stack.last[:free_spacing]
        end

        emit(:group, :close, *text(data, ts, te))
      end
    };

    # Group backreference, named and numbered
    # ------------------------------------------------------------------------
    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
      case text = text(data, ts, te).first
      when /^\\([gk])(<>|'')/ # angle brackets
        validation_error(:backref, 'ref/call', 'ref ID is empty')

      when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :name_ref_ab, text, ts, te)
        else
          emit(:backref, :name_call_ab, text, ts, te)
        end

      when /^\\([gk])'[^\d+-]\w*'/ #single quotes
        if $1 == 'k'
          emit(:backref, :name_ref_sq, text, ts, te)
        else
          emit(:backref, :name_call_sq, text, ts, te)
        end

      when /^\\([gk])<\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_ref_ab, text, ts, te)
        else
          emit(:backref, :number_call_ab, text, ts, te)
        end

      when /^\\([gk])'\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_ref_sq, text, ts, te)
        else
          emit(:backref, :number_call_sq, text, ts, te)
        end

      when /^\\(?:g<\+|g<-|(k)<-)\d+>/ # angle-brackets
        if $1 == 'k'
          emit(:backref, :number_rel_ref_ab, text, ts, te)
        else
          emit(:backref, :number_rel_call_ab, text, ts, te)
        end

      when /^\\(?:g'\+|g'-|(k)'-)\d+'/ # single quotes
        if $1 == 'k'
          emit(:backref, :number_rel_ref_sq, text, ts, te)
        else
          emit(:backref, :number_rel_call_sq, text, ts, te)
        end

      when /^\\k<[^\d+\-]\w*[+\-]\d+>/ # angle-brackets
        emit(:backref, :name_recursion_ref_ab, text, ts, te)

      when /^\\k'[^\d+\-]\w*[+\-]\d+'/ # single-quotes
        emit(:backref, :name_recursion_ref_sq, text, ts, te)

      when /^\\([gk])<[+\-]?\d+[+\-]\d+>/ # angle-brackets
        emit(:backref, :number_recursion_ref_ab, text, ts, te)

      when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
        emit(:backref, :number_recursion_ref_sq, text, ts, te)

      end
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text = text(data, ts, te).first
      when '?' ;  emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text = text(data, ts, te).first
      when '*' ;  emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text = text(data, ts, te).first
      when '+' ;  emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    quantifier_interval  @err(premature_end_error) {
      emit(:quantifier, :interval, *text(data, ts, te))
    };

    # Escaped sequences
    # ------------------------------------------------------------------------
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    comment {
      if free_spacing
        emit(:free_space, :comment, *text(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    space+ {
      if free_spacing
        emit(:free_space, :whitespace, *text(data, ts, te))
      else
        append_literal(data, ts, te)
      end
    };

    # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
    # except meta characters.
    # ------------------------------------------------------------------------
    (ascii_print -- space)+    |
    ascii_nonprint+ |
    utf8_2_byte+    |
    utf8_3_byte+    |
    utf8_4_byte+    {
      append_literal(data, ts, te)
    };

  *|;
}%%

# THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
# This file was generated from lib/regexp_parser/scanner/scanner.rl

class Regexp::Scanner
  # General scanner error (catch all)
  class ScannerError < StandardError; end

  # Base for all scanner validation errors
  class ValidationError < StandardError
    def initialize(reason)
      super reason
    end
  end

  # Unexpected end of pattern
  class PrematureEndError < ScannerError
    def initialize(where = '')
      super "Premature end of pattern at #{where}"
    end
  end

  # Invalid sequence format. Used for escape sequences, mainly.
  class InvalidSequenceError < ValidationError
    def initialize(what = 'sequence', where = '')
      super "Invalid #{what} at #{where}"
    end
  end

  # Invalid group. Used for named groups.
  class InvalidGroupError < ValidationError
    def initialize(what, reason)
      super "Invalid #{what}, #{reason}."
    end
  end

  # Invalid groupOption. Used for inline options.
  class InvalidGroupOption < ValidationError
    def initialize(option, text)
      super "Invalid group option #{option} in #{text}"
    end
  end

  # Invalid back reference. Used for name a number refs/calls.
  class InvalidBackrefError < ValidationError
    def initialize(what, reason)
      super "Invalid back reference #{what}, #{reason}"
    end
  end

  # The property name was not recognized by the scanner.
  class UnknownUnicodePropertyError < ValidationError
    def initialize(name)
      super "Unknown unicode character property name #{name}"
    end
  end

  # Scans the given regular expression text, or Regexp object and collects the
  # emitted token into an array that gets returned at the end. If a block is
  # given, it gets called for each emitted token.
  #
  # This method may raise errors if a syntax error is encountered.
  # --------------------------------------------------------------------------
  def self.scan(input_object, &block)
    new.scan(input_object, &block)
  end

  def scan(input_object, &block)
    self.literal = nil
    stack = []

    if input_object.is_a?(Regexp)
      input = input_object.source
      self.free_spacing = (input_object.options & Regexp::EXTENDED != 0)
    else
      input = input_object
      self.free_spacing = false
    end

    data  = input.unpack("c*") if input.is_a?(String)
    eof   = data.length

    self.tokens = []
    self.block  = block_given? ? block : nil

    self.in_group = false
    self.group_depth = 0
    self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]

    in_set = false
    set_depth = 0
    in_conditional = false
    conditional_depth = 0
    conditional_stack = []

    %% write data;
    %% write init;
    %% write exec;

    # to avoid "warning: assigned but unused variable - testEof"
    testEof = testEof

    if cs == re_scanner_error
      text = ts ? copy(data, ts-1..-1) : data.pack('c*')
      raise ScannerError.new("Scan error at '#{text}'")
    end

    raise PrematureEndError.new("(missing group closing paranthesis) "+
          "[#{in_group}:#{group_depth}]") if in_group
    raise PrematureEndError.new("(missing set closing bracket) "+
          "[#{in_set}:#{set_depth}]") if in_set

    # when the entire expression is a literal run
    emit_literal if literal

    tokens
  end

  # lazy-load property maps when first needed
  require 'yaml'
  PROP_MAPS_DIR = File.expand_path('../scanner/properties', __FILE__)

  def self.short_prop_map
    @short_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/short.yml")
  end

  def self.long_prop_map
    @long_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/long.yml")
  end

  # Emits an array with the details of the scanned pattern
  def emit(type, token, text, ts, te)
    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"

    emit_literal if literal

    if block
      block.call type, token, text, ts, te
    end

    tokens << [type, token, text, ts, te]
  end

  private

  attr_accessor :tokens, :literal, :block,
                :in_group, :group_depth,
                :free_spacing, :spacing_stack

  # Copy from ts to te from data as text
  def copy(data, range)
    data[range].pack('c*')
  end

  # Copy from ts to te from data as text, returning an array with the text
  #  and the offsets used to copy it.
  def text(data, ts, te, soff = 0)
    [copy(data, ts-soff..te-1), ts-soff, te]
  end

  # Appends one or more characters to the literal buffer, to be emitted later
  # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
  def append_literal(data, ts, te)
    self.literal = literal || []
    literal << text(data, ts, te)
  end

  # Emits the literal run collected by calls to the append_literal method,
  # using the total start (ts) and end (te) offsets of the run.
  def emit_literal
    ts, te = literal.first[1], literal.last[2]
    text = literal.map {|t| t[0]}.join

    text.force_encoding('utf-8') if text.respond_to?(:force_encoding)

    self.literal = nil
    emit(:literal, :literal, text, ts, te)
  end

  def emit_options(text, ts, te)
    token = nil

    # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
    text =~ /\(\?([mixdau]*)(-(?:[mix]*))*(:)?/
    positive, negative, group_local = $1, $2, $3

    if positive.include?('x')
      self.free_spacing = true
    end

    # If the x appears in both, treat it like ruby does, the second cancels
    # the first.
    if negative && negative.include?('x')
      self.free_spacing = false
    end

    if group_local
      spacing_stack << {:free_spacing => free_spacing, :depth => group_depth}
      token = :options
    else
      # switch for parent group level
      spacing_stack.last[:free_spacing] = free_spacing
      token = :options_switch
    end

    emit(:group, token, text, ts, te)
  end

  def emit_meta_control_sequence(data, ts, te, token)
    if data.last < 0x00 || data.last > 0x7F
      validation_error(:sequence, 'escape', token.to_s)
    end
    emit(:escape, token, *text(data, ts, te, 1))
  end

  # Centralizes and unifies the handling of validation related
  # errors.
  def validation_error(type, what, reason)
    case type
    when :group
      error = InvalidGroupError.new(what, reason)
    when :backref
      error = InvalidBackrefError.new(what, reason)
    when :sequence
      error = InvalidSequenceError.new(what, reason)
    end

    raise error # unless @@config.validation_ignore
  end
end # module Regexp::Scanner


Ragel: make sure ID is an identifier

Ragel: make sure ID is an identifier

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( 'ID' = identifier | key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end