Fix ref character handing in mixed segment

– Encountering a ref character with gap in alt when processing a mixed segment, a new segment should be started b.c. the previously encountered alt characters should be aligned to the previous ref character.

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(match_continue_g))		|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(match_continue_g))		|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(mixed_continue_r))		|	# Adding only to the reference does not change same to diff.
																	# Correct b.c. the machine exits in the following two states.
				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}


Ragel: some text

Ragel: some text

# Ragel lexer for VCF-header
#
# This is a partial lexer for the VCF header format. Bio-vcf uses this
# to generate meta information in (for example) JSON format. The
# advantage of using a full state engine is that it allows for easy
# parsing of key-value pairs with syntax checking and, for example,
# escaped quotes in quoted string values. This edition validates ID and
# Number fields only.

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Ragel lexer for VCF-header
#
# This is a partial lexer for the VCF file format. Bio-vcf uses that
# to generate meta information in (for example) JSON format. The
# advantage of using a full state engine is that it allows for easy
# parsing of key-value pairs with syntax checking and, for example,
# escaped quotes in quoted string values. Currently we validate ID and
# Number fields only. If should not be hard to continue adding syntax
# checking features.

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:kw,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );

  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  begin
    res = {}
    # p values
    values.each_slice(2) do | a,b |
      # p '*',a,b
      res[a[1]] = b[1]
      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
    end
  rescue
    print "ERROR: "
    p values
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Fix out-of-tree builds

Fix out-of-tree builds

#include "sys/configfile.hpp"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <err.h>
#include <stdarg.h>

#define MAX_WARNINGS 10

#define WARN(...) \
    do { \
        configFileWarn(path, lineno, __VA_ARGS__); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

static void configFileWarn(char const *path, unsigned int lineno,
                           char const *msg, ...)
{
    va_list ap;
    fprintf(stderr, "%s:%u: ", path, lineno);
    va_start(ap, msg);
    vfprintf(stderr, msg, ap);
    va_end(ap);
    putc('\n', stderr);
}

#define WARNC(msg, c) \
    do { \
        configFileWarnC(path, lineno, msg, *p); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

%%{
    machine configfile;
    write data;

    action putc {
        tmp[tmppos++] = *p;
    }
    action pute {
        switch (*p) {
        case 'n': tc = '\n'; break;
        case '\\': tc = '\\'; break;
        case '"': tc = '"'; break;
        default:
            tc = *p;
            WARN("Invalid escape sequence: \"\\%c\"", (int)tc);
            break;
        }
        tmp[tmppos++] = tc;
    }
    action startsec {
        secend = 0;
        keyend = 0;
        valend = 0;
        tmppos = 0;
    }
    action startvar {
        keyend = 0;
        valend = 0;
        tmppos = secend;
        quoted = false;
    }
    action setquoted {
        quoted = true;
    }
    action endsec {
        secend = tmppos;
    }
    action endkey {
        keyend = tmppos;
    }
    action endval {
        valend = tmppos;
    }
    action nextline {
        lineno += 1;
    }
    action endvarline {
        if (!secend)
            WARN("Variable appears outside config section");
        else {
            if (!quoted) {
                while (valend > keyend && tmp[valend - 1] == ' ')
                    valend--;
            }
            char *d = alloc(valend + 3),
                *sec = d, *key = d + secend + 1, *val = d + keyend + 2;
            memcpy(sec, tmp, secend);
            sec[secend] = '\0';
            memcpy(key, tmp + secend, keyend - secend);
            key[keyend - secend] = '\0';
            memcpy(val, tmp + keyend, valend - keyend);
            val[valend - keyend] = '\0';
            Var v = { sec, key, val };
            vars_.push_back(v);
            sorted_ = false;
        }
    }
    action error {
        WARN("Syntax error in configuration file");
        fhold;
        fgoto eaterr;
    }
    action resume {
        fhold;
        fgoto main;
    }

    lbreak = (('\r' '\n'?) | '\n') $(line,1) %(line,0) >nextline ;
    ws = (' ' | '\t')* ;
    comment = (';' | '#') (any - ('\n' | '\r'))* ;

    csec = (alnum | '.' | '-')+ @putc %endsec ;
    ckey = (alnum | '.' | '-')+ @putc %endkey ;
    cvunquotc = print - ('"' | '\\' | '#' | ';') ;
    cvunquot = ((cvunquotc - ' ') cvunquotc**) @putc ;
    cvescape = '\\' print %to(pute);
    cvplain = (print - ('"' | '\\')) @putc ;
    cvquotc = cvplain | cvescape ;
    cvquot = ('"' cvquotc* '"') >setquoted ;
    cval = (cvunquot | cvquot)? %endval ;

    varline = (ckey ws '=' ws cval ws) >startvar %endvarline ;
    secline = ('[' ws csec ws ']' ws) >startsec ;

    line = ws (varline | secline)? comment? ;

    main := (line lbreak)* line? $!error ;

    eaterr := (any - ('\n' | '\r'))* $!resume ;
}%%

void ConfigFile::read(char const *path)
{
    unsigned int warncount = 0;
    size_t bufsz = 1024, tmpsz = bufsz * 2;
    char *buf = 0, *tmp = 0;
    FILE *f = 0;
    f = fopen(path, "rb");
    if (!f) goto error;
    buf = reinterpret_cast<char *>(malloc(bufsz));
    if (!buf) goto error;
    tmp = reinterpret_cast<char *>(malloc(tmpsz));
    if (!tmp) goto error;
    try {
        int cs;
        size_t tmppos = 0, secend = 0, keyend = 0, valend = 0, amt;
        unsigned int lineno = 1;
        unsigned char tc;
        bool quoted = false;
        %% write init;
        do {
            amt = fread(buf, 1, sizeof(buf), f);
            char *p = buf, *pe = buf + amt, *eof = NULL;
            if (!amt) {
                if (ferror(f))
                    goto error;
                eof = pe;
            }
            if (tmppos + amt > tmpsz) {
                fprintf(stderr, "%s:%u: Line too long\n", path, lineno);
                break;
            }
            %% write exec;
        } while (amt);
    } catch (...) {
        free(tmp);
        free(buf);
        fclose(f);
        throw;
    }
    goto done;
error:
    warn("Error reading config file '%s'", path);
done:
    if (warncount == MAX_WARNINGS)
        fprintf(stderr, "%s: Too many warnings, aborting\n", path);
    free(tmp);
    free(buf);
    if (f) fclose(f);
    return;
}

#include "configfile.hpp"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <err.h>
#include <stdarg.h>

#define MAX_WARNINGS 10

#define WARN(...) \
    do { \
        configFileWarn(path, lineno, __VA_ARGS__); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

static void configFileWarn(char const *path, unsigned int lineno,
                           char const *msg, ...)
{
    va_list ap;
    fprintf(stderr, "%s:%u: ", path, lineno);
    va_start(ap, msg);
    vfprintf(stderr, msg, ap);
    va_end(ap);
    putc('\n', stderr);
}

#define WARNC(msg, c) \
    do { \
        configFileWarnC(path, lineno, msg, *p); \
        if (++warncount > MAX_WARNINGS) goto done; \
    } while (0)

%%{
    machine configfile;
    write data;

    action putc {
        tmp[tmppos++] = *p;
    }
    action pute {
        switch (*p) {
        case 'n': tc = '\n'; break;
        case '\\': tc = '\\'; break;
        case '"': tc = '"'; break;
        default:
            tc = *p;
            WARN("Invalid escape sequence: \"\\%c\"", (int)tc);
            break;
        }
        tmp[tmppos++] = tc;
    }
    action startsec {
        secend = 0;
        keyend = 0;
        valend = 0;
        tmppos = 0;
    }
    action startvar {
        keyend = 0;
        valend = 0;
        tmppos = secend;
        quoted = false;
    }
    action setquoted {
        quoted = true;
    }
    action endsec {
        secend = tmppos;
    }
    action endkey {
        keyend = tmppos;
    }
    action endval {
        valend = tmppos;
    }
    action nextline {
        lineno += 1;
    }
    action endvarline {
        if (!secend)
            WARN("Variable appears outside config section");
        else {
            if (!quoted) {
                while (valend > keyend && tmp[valend - 1] == ' ')
                    valend--;
            }
            char *d = alloc(valend + 3),
                *sec = d, *key = d + secend + 1, *val = d + keyend + 2;
            memcpy(sec, tmp, secend);
            sec[secend] = '\0';
            memcpy(key, tmp + secend, keyend - secend);
            key[keyend - secend] = '\0';
            memcpy(val, tmp + keyend, valend - keyend);
            val[valend - keyend] = '\0';
            Var v = { sec, key, val };
            vars_.push_back(v);
            sorted_ = false;
        }
    }
    action error {
        WARN("Syntax error in configuration file");
        fhold;
        fgoto eaterr;
    }
    action resume {
        fhold;
        fgoto main;
    }

    lbreak = (('\r' '\n'?) | '\n') $(line,1) %(line,0) >nextline ;
    ws = (' ' | '\t')* ;
    comment = (';' | '#') (any - ('\n' | '\r'))* ;

    csec = (alnum | '.' | '-')+ @putc %endsec ;
    ckey = (alnum | '.' | '-')+ @putc %endkey ;
    cvunquotc = print - ('"' | '\\' | '#' | ';') ;
    cvunquot = ((cvunquotc - ' ') cvunquotc**) @putc ;
    cvescape = '\\' print %to(pute);
    cvplain = (print - ('"' | '\\')) @putc ;
    cvquotc = cvplain | cvescape ;
    cvquot = ('"' cvquotc* '"') >setquoted ;
    cval = (cvunquot | cvquot)? %endval ;

    varline = (ckey ws '=' ws cval ws) >startvar %endvarline ;
    secline = ('[' ws csec ws ']' ws) >startsec ;

    line = ws (varline | secline)? comment? ;

    main := (line lbreak)* line? $!error ;

    eaterr := (any - ('\n' | '\r'))* $!resume ;
}%%

void ConfigFile::read(char const *path)
{
    unsigned int warncount = 0;
    size_t bufsz = 1024, tmpsz = bufsz * 2;
    char *buf = 0, *tmp = 0;
    FILE *f = 0;
    f = fopen(path, "rb");
    if (!f) goto error;
    buf = reinterpret_cast<char *>(malloc(bufsz));
    if (!buf) goto error;
    tmp = reinterpret_cast<char *>(malloc(tmpsz));
    if (!tmp) goto error;
    try {
        int cs;
        size_t tmppos = 0, secend = 0, keyend = 0, valend = 0, amt;
        unsigned int lineno = 1;
        unsigned char tc;
        bool quoted = false;
        %% write init;
        do {
            amt = fread(buf, 1, sizeof(buf), f);
            char *p = buf, *pe = buf + amt, *eof = NULL;
            if (!amt) {
                if (ferror(f))
                    goto error;
                eof = pe;
            }
            if (tmppos + amt > tmpsz) {
                fprintf(stderr, "%s:%u: Line too long\n", path, lineno);
                break;
            }
            %% write exec;
        } while (amt);
    } catch (...) {
        free(tmp);
        free(buf);
        fclose(f);
        throw;
    }
    goto done;
error:
    warn("Error reading config file '%s'", path);
done:
    if (warncount == MAX_WARNINGS)
        fprintf(stderr, "%s: Too many warnings, aborting\n", path);
    free(tmp);
    free(buf);
    if (f) fclose(f);
    return;
}


Deal with idiotic C types + format string specifications.

// vim: filetype=ragel

%%{

  machine pack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    rest = true;
  }

  action platform {
    platform = true;
  }

  action check_size {
    stop = rest ? size() : index + count;
    if(stop > size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action C {
    PACK_INT_ELEMENTS(MASK_BYTE);
  }

  action S {
    PACK_INT_ELEMENTS(MASK_16BITS);
  }

  action I {
    PACK_INT_ELEMENTS(MASK_32BITS);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      PACK_INT_ELEMENTS(MASK_32BITS);
#else
      PACK_LONG_ELEMENTS(MASK_64BITS);
#endif
    } else {
      PACK_INT_ELEMENTS(MASK_32BITS);
    }
  }

  action n {
    PACK_INT_ELEMENTS(BE_MASK_16BITS);
  }

  action N {
    PACK_INT_ELEMENTS(BE_MASK_32BITS);
  }

  action v {
    PACK_INT_ELEMENTS(LE_MASK_16BITS);
  }

  action V {
    PACK_INT_ELEMENTS(LE_MASK_32BITS);
  }

  action Q {
    PACK_LONG_ELEMENTS(MASK_64BITS);
  }

  action X {
#define INVALID_MOVE_ERROR_SIZE 48

    if(rest) count = 0;

    if(count > str.size()) {
      char invalid_move_msg[INVALID_MOVE_ERROR_SIZE];
      snprintf(invalid_move_msg, INVALID_MOVE_ERROR_SIZE,
               "X%d exceeds length of string", (int)count);
      Exception::argument_error(state, invalid_move_msg);
    }

    str = str.substr(0, str.size() - count);
  }

  action x {
    if(rest) count = 0;

    str.append(count, '\0');
  }

  action at {
    if(rest) count = 1;

    if(count > str.size()) {
      str.append(count - str.size(), '\0');
    } else {
      str = str.substr(0, count);
    }
  }

  action fail {
    return force_as<String>(Primitives::failure());
  }

  action non_native_error {
#define NON_NATIVE_ERROR_SIZE 36

    char non_native_msg[NON_NATIVE_ERROR_SIZE];
    snprintf(non_native_msg, NON_NATIVE_ERROR_SIZE,
             "'%c' allowed only after types sSiIlL", *p);
    Exception::argument_error(state, non_native_msg);
  }

  action done {
    return String::create(state, str.c_str(), str.size());
  }
}%%

// vim: filetype=ragel

%%{

  machine pack;

  action start {
    count = 1;
    rest = false;
    platform = false;
  }

  action start_digit {
    count = fc - '0';
  }

  action count {
    count = count * 10 + (fc - '0');
  }

  action rest {
    rest = true;
  }

  action platform {
    platform = true;
  }

  action check_size {
    stop = rest ? size() : index + count;
    if(stop > size()) {
      Exception::argument_error(state, "too few arguments");
    }
  }

  action C {
    PACK_INT_ELEMENTS(MASK_BYTE);
  }

  action S {
    PACK_INT_ELEMENTS(MASK_16BITS);
  }

  action I {
    PACK_INT_ELEMENTS(MASK_32BITS);
  }

  action L {
    if(platform) {
#if RBX_SIZEOF_LONG == 4
      PACK_INT_ELEMENTS(MASK_32BITS);
#else
      PACK_LONG_ELEMENTS(MASK_64BITS);
#endif
    } else {
      PACK_INT_ELEMENTS(MASK_32BITS);
    }
  }

  action n {
    PACK_INT_ELEMENTS(BE_MASK_16BITS);
  }

  action N {
    PACK_INT_ELEMENTS(BE_MASK_32BITS);
  }

  action v {
    PACK_INT_ELEMENTS(LE_MASK_16BITS);
  }

  action V {
    PACK_INT_ELEMENTS(LE_MASK_32BITS);
  }

  action Q {
    PACK_LONG_ELEMENTS(MASK_64BITS);
  }

  action X {
#define INVALID_MOVE_ERROR_SIZE 48

    if(rest) count = 0;

    if(count > str.size()) {
      char invalid_move_msg[INVALID_MOVE_ERROR_SIZE];
      snprintf(invalid_move_msg, INVALID_MOVE_ERROR_SIZE,
               "X%ld request exceeds length of string", count);
      Exception::argument_error(state, invalid_move_msg);
    }

    str = str.substr(0, str.size() - count);
  }

  action x {
    if(rest) count = 0;

    str.append(count, '\0');
  }

  action at {
    if(rest) count = 1;

    if(count > str.size()) {
      str.append(count - str.size(), '\0');
    } else {
      str = str.substr(0, count);
    }
  }

  action fail {
    return force_as<String>(Primitives::failure());
  }

  action non_native_error {
#define NON_NATIVE_ERROR_SIZE 36

    char non_native_msg[NON_NATIVE_ERROR_SIZE];
    snprintf(non_native_msg, NON_NATIVE_ERROR_SIZE,
             "'%c' allowed only after types sSiIlL", *p);
    Exception::argument_error(state, non_native_msg);
  }

  action done {
    return String::create(state, str.c_str(), str.size());
  }
}%%


Now it even compiles

Now it even compiles

package dns

// Parse private key files

import (
    "os"
    "strings"
)

%%{
        machine k;
        write data;
}%%

func Kparse(data string) (m map[string]string, err os.Error) {
        cs, p, pe, eof := 0, 0, len(data), len(data)
        mark := 0
        key := ""

        %%{
                action mark      { mark = p }
                action setKey    { key = strings.ToLower(data[mark:p]) }
                action setValue  { m[key] = data[mark:p] }

                key = (
                      ('Private-key-format'i)
                    | ('Algorithm'i)
                    | ('Modules'i)
                    | ('PublicExponent'i)
                    | ('PrivateExponent'i)
                    | ('Prime1'i)
                    | ('Prime2'i)
                    | ('Exponent1'i)
                    | ('Exponent2'i)
                    | ('Coefficient'i)
                    | ('Created'i)
                    | ('Publish'i)
                    | ('Activate'i)
                ) %setKey;

                value = any+ %setValue;

                line = key /: ?/ value;
                main := line+;

                write init;
                write exec;
        }%%

        if cs < z_first_final {
                // No clue what I'm doing what so ever
                if p == pe {
                        //return nil, os.ErrorString("unexpected eof")
                        return nil, nil
                } else {
                        //return nil, os.ErrorString(fmt.Sprintf("error at position %d", p))
                        return nil, nil
                }
        }
        return
}

package dns

// Parse private key files

import (
    "os"
    "strings"
)

%%{
        machine k;
        write data;
}%%

func Kparse(data string) (m map[string]string, err os.Error) {
        cs, p, pe, eof := 0, 0, len(data), len(data)
        mark := 0
        key := ""

        %%{
                action mark      { mark = p }
                action setKey    { key = strings.ToLower(data[mark:p]) }
                action setValue  { m[key] = data[mark:p] }

                key = (
                      ('Private-key-format'i)
                    | ('Algorithm'i)
                    | ('Modules'i)
                    | ('PublicExponent'i)
                    | ('PrivateExponent'i)
                    | ('Prime1'i)
                    | ('Prime2'i)
                    | ('Exponent1'i)
                    | ('Exponent2'i)
                    | ('Coefficient'i)
                    | ('Created'i)
                    | ('Publish'i)
                    | ('Activate'i)
                );

                value = any+;

                line = key /: ?/ value;
                main := line+;

                write init;
                write exec;
        }%%

        if cs < z_first_final {
                // No clue what I'm doing what so ever
                if p == pe {
                        //return nil, os.ErrorString("unexpected eof")
                        return nil, nil
                } else {
                        //return nil, os.ErrorString(fmt.Sprintf("error at position %d", p))
                        return nil, nil
                }
        }
        return r ,nil
}