Fix mixed segment handling

Fix mixed segment handling

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTNacgtn];
			ntg			= nt | '-';

			a_			= [Aa];
			c_			= [Cc];
			g_			= [Gg];
			t_			= [Tt];
			n_			= [Nn];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/i | /CC/i | /GG/i | /TT/i | /NN/i;
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(match_continue_g))		|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(mixed_continue_r))		|	# Adding only to the reference does not change same to diff.
																	# Correct b.c. the machine exits in the following two states.
				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTNacgtn];
			ntg			= nt | '-';

			a_			= [Aa];
			c_			= [Cc];
			g_			= [Gg];
			t_			= [Tt];
			n_			= [Nn];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/i | /CC/i | /GG/i | /TT/i | /NN/i;
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_match_b))		|
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(match_continue_g))		|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(mixed_continue_r))		|	# Adding only to the reference does not change same to diff.
																	# Correct b.c. the machine exits in the following two states.
				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}


Fixed UTF-8 bug in lexer

Fixed UTF-8 bug in lexer

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

enum {
    kTQStrTypeSimple,
    kTQStrTypeLeftOrMiddle,
    kTQStrTypeRight
};
#define PushStr(dblQuot, type) \
    [strings addObject:[NSDictionary dictionaryWithObjectsAndKeys: \
        [NSNumber numberWithBool:dblQuot], @"dblQuot", \
        [NSNumber numberWithInt:type], @"type", \
        [NSMutableData data], @"value", nil]];
#define PopStr() ({ id last = [Str() retain]; [strings removeLastObject]; [last autorelease]; })
#define StrData() [[strings lastObject] objectForKey:@"value"]
#define Str() [[[NSMutableString alloc] initWithData:StrData() encoding:NSUTF8StringEncoding] autorelease]
#define IsDblQuot() [[[strings lastObject] objectForKey:@"dblQuot"] boolValue]
#define IsRstr() ([[[strings lastObject] objectForKey:@"type"] intValue] == kTQStrTypeRight)
#define IsSimpleStr() ([[[strings lastObject] objectForKey:@"type"] intValue] == kTQStrTypeSimple)

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    int tokenId_ = tokenId; \
    id val_ = val; \
    /*NSLog(@"emitting %d = '%@' on line: %d", tokenId, val_, parserState.currentLine);*/ \
    Parse(parser, tokenId_, [TQToken withId:tokenId value:val_ line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken(tokenId, [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    lGuillemet  = 0xC2 0xAB; # «
    rGuillemet  = 0xC2 0xBB; # »
    specialChar = 0xC2 | '"' | "'" | '.' | ',' | ';' | ':' | '|' | '#' | '@' | '~' | '`' | '{' | '}' | '[' | ']' | '(' | ')' | '+' | '-' | '*' | '/' | '%' | '=' | '<' | '>' | '^' | '\\' | '\r' | '\n' | space;

    char        = (any - specialChar) | (0xC2 ^(0xAB|0xBB));
    constant    = '_'* uupper char*;
    identifier  = (char -- (uupper|udigit)) char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    constStr    = "#" (char | [:@#~+\-*/%=<>^])+;
    selector    = char* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\n'                             => { [StrData() appendBytes:"\n" length:1]; IncrementLine();        };
    '\\n'                            => { [StrData() appendBytes:"\n" length:1];                         };
    '\\t'                            => { [StrData() appendBytes:"\t" length:1];                         };
    '\\r'                            => { [StrData() appendBytes:"\r" length:1];                         };
    '\\"'                            => { [StrData() appendBytes:"\"" length:1];                         };
    "\\'"                            => { [StrData() appendBytes:"'"  length:1];                         };
    "\\\\"                           => { [StrData() appendBytes:"\\" length:1];                         };
    "\\" lGuillemet                  => { [StrData() appendBytes:"«"  length:2];                         };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        NSString *strVal = [NSString stringWithFormat:@"%c", (int)code];
        [StrData() appendData:[strVal dataUsingEncoding:NSUTF8StringEncoding]];
    };
    "\\" any                         => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                };

    lGuillemet                       => {
        TQAssert(!IsSimpleStr(), @"Interpolation is not allowed in constant strings");
        _EmitToken(IsRstr() ? MSTR : LSTR, Str());
        fret;
    };
    '"' term => {
        if(IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTRNL
                                      : (IsRstr() ? RSTRNL : STRNL);
            _EmitToken(tokId, PopStr());
            BacktrackTerm();
            fret;
        } else
            [StrData() appendBytes:@"\"" length:1];
    };
    '"' => {
        if(IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTR
                                      : (IsRstr() ? RSTR : STR);
            _EmitToken(tokId, PopStr());
            fret;
        } else
            [StrData() appendBytes:@"\"" length:1];
    };
    "'" term => {
        if(!IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTRNL
                                      : (IsRstr() ? RSTRNL : STRNL);
            _EmitToken(tokId, PopStr());
            BacktrackTerm();
            fret;
        } else
            [StrData() appendBytes:@"'" length:1];
    };
    "'" => {
        if(!IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTR
                                      : (IsRstr() ? RSTR : STR);
            _EmitToken(tokId, PopStr());
            fret;
        } else
            [StrData() appendBytes:@"'" length:1];
    };

    any => { [StrData() appendBytes:ts length:1];                                                         };
*|;

regex := |*
    '\n'                             => { [StrData() appendBytes:"\n"  length:1]; IncrementLine();        };
    "\\/"                            => { [StrData() appendBytes:"\\/" length:1];                         };
    "\\"                             => { [StrData() appendBytes:"\\"  length:1];                         };
    regexCont                        => { [StrData() appendBytes:ts    length:1];                         };
    "/" [im]* term                   => { [StrData() appendBytes:"/"   length:1];
                                          _EmitToken(REGEXNL, PopStr());
                                          BacktrackTerm(); fret;                                          };
    "/" [im]*                        => { [StrData() appendBytes:"/" length:1];
                                          _EmitToken(REGEX, PopStr()); fret;                              };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            PushStr(false, kTQStrTypeLeftOrMiddle);
            [StrData() appendBytes:"/" length:1];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "@"                              => { EmitToken(ATMARK);                                              };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    "#" '"'                          => { PushStr(YES, kTQStrTypeSimple); fcall string;                   };
    "#" "'"                          => { PushStr(NO, kTQStrTypeSimple);  fcall string;                   };
    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { PushStr(YES, kTQStrTypeLeftOrMiddle); fcall string;             };
    "'"                              => { PushStr(NO, kTQStrTypeLeftOrMiddle);  fcall string;             };
    rGuillemet                       => {
        BOOL dbl = IsDblQuot();
        PopStr();
        PushStr(dbl, kTQStrTypeRight);
        fcall string;
    };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableArray *strings = [NSMutableArray array];

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

enum {
    kTQStrTypeSimple,
    kTQStrTypeLeftOrMiddle,
    kTQStrTypeRight
};
#define PushStr(dblQuot, type) \
    [strings addObject:[NSDictionary dictionaryWithObjectsAndKeys: \
        [NSNumber numberWithBool:dblQuot], @"dblQuot", \
        [NSNumber numberWithInt:type], @"type", \
        [NSMutableString string], @"value", nil]];
#define PopStr() ({ id last = [Str() retain]; [strings removeLastObject]; [last autorelease]; })
#define Str() [[strings lastObject] objectForKey:@"value"]
#define IsDblQuot() [[[strings lastObject] objectForKey:@"dblQuot"] boolValue]
#define IsRstr() ([[[strings lastObject] objectForKey:@"type"] intValue] == kTQStrTypeRight)
#define IsSimpleStr() ([[[strings lastObject] objectForKey:@"type"] intValue] == kTQStrTypeSimple)

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    int tokenId_ = tokenId; \
    id val_ = val; \
    /*NSLog(@"emitting %d = '%@' on line: %d", tokenId, val_, parserState.currentLine);*/ \
    Parse(parser, tokenId_, [TQToken withId:tokenId value:val_ line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken(tokenId, [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    lGuillemet  = 0xC2 0xAB; # «
    rGuillemet  = 0xC2 0xBB; # »
    specialChar = 0xC2 | '"' | "'" | '.' | ',' | ';' | ':' | '|' | '#' | '@' | '~' | '`' | '{' | '}' | '[' | ']' | '(' | ')' | '+' | '-' | '*' | '/' | '%' | '=' | '<' | '>' | '^' | '\\' | '\r' | '\n' | space;

    char        = (any - specialChar) | (0xC2 ^(0xAB|0xBB));
    constant    = '_'* uupper char*;
    identifier  = (char -- (uupper|udigit)) char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    constStr    = "#" (char | [:@#~+\-*/%=<>^])+;
    selector    = char* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    '\\n'                            => { [Str() appendString:@"\n"];                                    };
    '\\t'                            => { [Str() appendString:@"\t"];                                    };
    '\\r'                            => { [Str() appendString:@"\r"];                                    };
    '\\"'                            => { [Str() appendString:@"\""];                                    };
    "\\'"                            => { [Str() appendString:@"'"];                                     };
    "\\\\"                           => { [Str() appendString:@"\\"];                                    };
    "\\" lGuillemet                  => { [Str() appendString:@"«"];                                     };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        [Str() appendFormat:@"%c", (int)code];
    };
    "\\" any                         => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                };

    lGuillemet                       => {
        TQAssert(!IsSimpleStr(), @"Interpolation is not allowed in constant strings");
        _EmitToken(IsRstr() ? MSTR : LSTR, Str());
        fret;
    };
    '"' term => {
        if(IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTRNL
                                      : (IsRstr() ? RSTRNL : STRNL);
            _EmitToken(tokId, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    '"' => {
        if(IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTR
                                      : (IsRstr() ? RSTR : STR);
            _EmitToken(tokId, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" term => {
        if(!IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTRNL
                                      : (IsRstr() ? RSTRNL : STRNL);
            _EmitToken(tokId, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" => {
        if(!IsDblQuot()) {
            int tokId = IsSimpleStr() ? CONSTSTR
                                      : (IsRstr() ? RSTR : STR);
            _EmitToken(tokId, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };

    any => { [Str() appendString:NSStr(0, 0)];                                                           };
*|;

regex := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    "\\/"                            => { [Str() appendString:@"\\/"];                                   };
    "\\"                             => { [Str() appendString:@"\\"];                                    };
    regexCont                        => { [Str() appendString:NSStr(0, 0)];                              };
    "/" [im]* term                   => { [Str() appendString:@"/"]; _EmitToken(REGEXNL, PopStr());
                                          BacktrackTerm(); fret;                                         };
    "/" [im]*                        => { [Str() appendString:@"/"]; _EmitToken(REGEX, PopStr()); fret;  };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            PushStr(false, kTQStrTypeLeftOrMiddle);
            [Str() appendString:@"/"];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "@"                              => { EmitToken(ATMARK);                                              };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    "#" '"'                          => { PushStr(YES, kTQStrTypeSimple); fcall string;                   };
    "#" "'"                          => { PushStr(NO, kTQStrTypeSimple);  fcall string;                   };
    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { PushStr(YES, kTQStrTypeLeftOrMiddle); fcall string;             };
    "'"                              => { PushStr(NO, kTQStrTypeLeftOrMiddle);  fcall string;             };
    rGuillemet                       => {
        BOOL dbl = IsDblQuot();
        PopStr();
        PushStr(dbl, kTQStrTypeRight);
        fcall string;
    };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableArray *strings = [NSMutableArray array];

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}


In progress table parsing refactoring

In progress table parsing refactoring

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  CellContent = ^('|' | EOL);

  # table is one or more rows
  # a row is space* then any number of cells followed by a '| space* EOL'
  # a cell is a '|' followed by anything that is not a '|'

  Cell = CellContent+ >begin_content '|' >store_cell_content | '|' >no_content;

  Row = space* '|' >initialize Cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);

  UnclosedRow = space* '|' >initialize Cell* space* CellContent+ %/bad_table_row %/store_table :>> (EOL+ space*);

  Table = (Row %store_row | UnclosedRow >set_bad_table_row_line %bad_table_row)+;

  main := Table %store_table;
}%%

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  cell_content = ^('|' | EOL);

  # table is one or more rows
  # a row is space* then any number of cells followed by a '| space* EOL'
  # a cell is a '|' followed by anything that is not a '|'

  Cell = cell_content+ >begin_content '|' >store_cell_content | '|' >no_content;

  table_row = space* '|' >initialize Cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);

  unclosed_table_row = space* '|' >initialize Cell* space* cell_content+ %/bad_table_row %/store_table :>> (EOL+ space*);

  Table = (table_row %store_row | unclosed_table_row >set_bad_table_row_line %bad_table_row)+ %store_table;

  main := Table;
}%%


Small whitespace fix

Small whitespace fix

/* Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>

#include "http11.h"

static int calc_offset(const char *fpc, const char *buf) {
    return fpc - buf;
}

static int calc_length(const char *fpc, const char *buf, const int offset) {
    return fpc - buf - offset;
}

static const char *create_ptr(const char *buf, const int offset) {
    return buf + offset;
}

%%{
    machine http_parser;

    action mark {
        parser->state->mark = calc_offset(fpc, data);
    }

    action request_method {
        if (parser->request_method != NULL) {
            parser->request_method(
                create_ptr(data, parser->state->mark),
                calc_length(fpc, data, parser->state->mark)
            );
        }
    }

    CRLF = ( "\r\n" | "\n" ) ;
    SP = " " ;

    tchar = ( "!" | "#" | "$" | "%" | "&" | "'" | "*" | "+" | "-" | "." | "^" |
              "_" | "`" | "|" | "~" | digit | alpha ) ;
    token = tchar+ ;

    method = token >mark %request_method ;
    request_target = ( any -- CRLF )+ ;
    http_version = "HTTP" "/" digit "." digit ;

    request_line = method SP request_target SP http_version CRLF ;
    http_message = ( request_line ) CRLF ;

main := http_message;

}%%

%% write data;

struct _HTTPParserState {
  int cs;
  int mark;
};

HTTPParser *HTTPParser_create() {
    HTTPParser *parser = malloc(sizeof(*parser));

    if (!parser) {
        return NULL;
    }

    parser->state = malloc(sizeof *parser->state);

    if (!parser->state) {
        HTTPParser_destroy(parser);
        return NULL;
    }

    return parser;
}

void HTTPParser_init(HTTPParser *parser) {
    %% access parser->state->;
    %% write init;

    parser->state->mark = 0;

    parser->finished = false;
    parser->error = 0;
}

size_t HTTPParser_execute(HTTPParser *parser, const char *data, size_t len, size_t off) {
    const char *p = data + off;
    const char *pe = data + len;

    %% access parser->state->;
    %% write exec;

    if (parser->state->cs == http_parser_error || parser->state->cs >= http_parser_first_final ) {
        parser->finished = true;

        if (parser->state->cs == http_parser_error && !parser->error) {
            parser->error = 1;
        }
    }

    return 1;
}

void HTTPParser_destroy(HTTPParser *parser) {
    if (parser) {
        free(parser->state);
        free(parser);
    }
}


json lexer: initialize return value.

json lexer: initialize return value.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "json_parser.h"
#include "json_obj.h"

void *jsonParseAlloc();
void jsonParse();
void jsonParseFree();

static void* _parser = 0;
static struct json_obj_state _state = { 0 };

void emit(int token, const char* s, char* ts, char* te)
{
    struct json_obj_pos pos = { .start = ts-s, .end = te-s };
    jsonParse(_parser, token, pos, &_state);
}

__attribute__((visibility("default")))
struct json_obj* json_lexer(const char* data)
{
    char* p = (char*)data;
    char* pe = p + strlen(p);
    char* eof = pe;
    int cs;
    char* ts = NULL;
    char* te = NULL;
    int act;

    (void)act;

    _parser = jsonParseAlloc(malloc);

    _state.obj = NULL;
    _state.line = 0;

    %%machine lexer;
    %%write data;
    (void)lexer_en_main;
    (void)lexer_error;
    (void)lexer_first_final;

    %%{
        action mark
        {
            mark();
        }

        newline = '\n';

        escape = '\\';
        escape_seq = escape (["/bfnrt] | escape | ('u' [0-9a-fA-F]{4}));
        string = '"' ((any - ('"' | escape) | escape_seq)*) '"';

        sign = '-'|'+';
        decimals = '.' digit+;
        exponent = [eE] sign? digit+;

        number = sign? digit+ decimals? exponent?;

        main := |*
            space-newline;
            newline => { _state.line++; };
            '{'     => { emit(TOK_LBRACE,   data, ts, te); };
            '}'     => { emit(TOK_RBRACE,   data, ts, te); };
            '['     => { emit(TOK_LBRACKET, data, ts, te); };
            ']'     => { emit(TOK_RBRACKET, data, ts, te); };
            ','     => { emit(TOK_COMMA,    data, ts, te); };
            ':'     => { emit(TOK_COLON,    data, ts, te); };
            'true'  => { emit(TOK_TRUE,     data, ts, te); };
            'false' => { emit(TOK_FALSE,    data, ts, te); };
            'null'  => { emit(TOK_NULL,     data, ts, te); };
            string  => { emit(TOK_STRING,   data, ts+1, te-1); };
            number  => { emit(TOK_NUMBER,   data, ts, te); };
        *|;

        write init;
        write exec;
    }%%

    jsonParseFree(_parser, free);
    return _state.obj;
}

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "json_parser.h"
#include "json_obj.h"

void *jsonParseAlloc();
void jsonParse();
void jsonParseFree();

static void* _parser = 0;
static struct json_obj_state _state = { 0 };

void emit(int token, const char* s, char* ts, char* te)
{
    struct json_obj_pos pos = { .start = ts-s, .end = te-s };
    jsonParse(_parser, token, pos, &_state);
}

__attribute__((visibility("default")))
struct json_obj* json_lexer(const char* data)
{
    char* p = (char*)data;
    char* pe = p + strlen(p);
    char* eof = pe;
    int cs;
    char* ts = NULL;
    char* te = NULL;
    int act;

    (void)act;

    _parser = jsonParseAlloc(malloc);

    %%machine lexer;
    %%write data;
    (void)lexer_en_main;
    (void)lexer_error;
    (void)lexer_first_final;

    %%{
        action mark
        {
            mark();
        }

        newline = '\n';

        escape = '\\';
        escape_seq = escape (["/bfnrt] | escape | ('u' [0-9a-fA-F]{4}));
        string = '"' ((any - ('"' | escape) | escape_seq)*) '"';

        sign = '-'|'+';
        decimals = '.' digit+;
        exponent = [eE] sign? digit+;

        number = sign? digit+ decimals? exponent?;

        main := |*
            space-newline;
            newline => { _state.line++; };
            '{'     => { emit(TOK_LBRACE,   data, ts, te); };
            '}'     => { emit(TOK_RBRACE,   data, ts, te); };
            '['     => { emit(TOK_LBRACKET, data, ts, te); };
            ']'     => { emit(TOK_RBRACKET, data, ts, te); };
            ','     => { emit(TOK_COMMA,    data, ts, te); };
            ':'     => { emit(TOK_COLON,    data, ts, te); };
            'true'  => { emit(TOK_TRUE,     data, ts, te); };
            'false' => { emit(TOK_FALSE,    data, ts, te); };
            'null'  => { emit(TOK_NULL,     data, ts, te); };
            string  => { emit(TOK_STRING,   data, ts+1, te-1); };
            number  => { emit(TOK_NUMBER,   data, ts, te); };
        *|;

        write init;
        write exec;
    }%%

    jsonParseFree(_parser, free);
    return _state.obj;
}


use access @; for using instance variables via ragel

%%{

  machine lexer;

  access @;

  T_KEYWORD_DEF = '->';
  T_KEYWORD_END = '<-';
  T_KEYWORD_OUT = 'stdout';
  T_DQUOTE      = '"';
  T_INT         = ('+'|'-')?[0-9]+;
  T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
  T_LITERAL     = [a-zA-Z][a-zA-Z_]+;
  T_ADD         = '+';
  T_SUB         = '-';
  T_MUL         = '*';
  T_DIV         = '/';
  T_MOD         = '%';
  T_EXP         = '^';
  T_LBR         = '(';
  T_RBR         = ')';
  T_EQ          = '=';
  T_COL         = ',';

  main := |*

    T_KEYWORD_DEF => {
      emit(:T_KEYWORD_DEF)
    };

    T_KEYWORD_END => {
      emit(:T_KEYWORD_END)
    };

    T_KEYWORD_OUT => {
      emit(:T_KEYWORD_OUT)
    };

    T_DQUOTE => {
      emit(:T_DQUOTE)
    };

    T_INT => {
      emit(:T_INT)
    };

    T_FLOAT => {
      emit(:T_FLOAT)
    };

    T_LITERAL => {
      emit(:T_LITERAL)
    };

    T_ADD => {
      emit(:T_ADD)
    };

    T_SUB => {
      emit(:T_SUB)
    };

    T_MUL => {
      emit(:T_MUL)
    };

    T_DIV => {
      emit(:T_DIV)
    };

    T_EXP => {
      emit(:T_EXP)
    };

    T_MOD => {
      emit(:T_MOD)
    };

    T_LBR => {
      emit(:T_LBR)
    };

    T_RBR => {
      emit(:T_RBR)
    };

    T_EQ => {
      emit(:T_EQ)
    };

    T_COL => {
      emit(:T_COL)
    };

    space;

  *|;

}%%

module CakeLang
  class Lexer
      CONVERSION = {
          :T_INT => :to_i,
          :T_FLOAT => :to_f
        }

      %% write data;

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      def initialize
        @tokens = []
      end

      def emit(symbol)
        # @ts, @te -> token start, token end
        value = @data[@ts..@te].pack("U*").strip

        if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
          value = value.send(CONVERSION[symbol])
        end

        @tokens << [symbol, value]
      end

      def lex(data)
        @data = data.unpack("U*")
        lexer_start = self.class.lexer_start
        eof = data.length

        %% write init;
        %% write exec;

        tokens = @tokens
      end
  end
end

%%{

  machine lexer;

  T_KEYWORD_DEF = '->';
  T_KEYWORD_END = '<-';
  T_KEYWORD_OUT = 'stdout';
  T_DQUOTE      = '"';
  T_INT         = ('+'|'-')?[0-9]+;
  T_FLOAT       = ('+'|'-')?[0-9]+'.'[0-9];
  T_LITERAL     = [a-zA-Z][a-zA-Z_]+;
  T_ADD         = '+';
  T_SUB         = '-';
  T_MUL         = '*';
  T_DIV         = '/';
  T_MOD         = '%';
  T_EXP         = '^';
  T_LBR         = '(';
  T_RBR         = ')';
  T_EQ          = '=';
  T_COL         = ',';

  main := |*

    T_KEYWORD_DEF => {
      emit(:T_KEYWORD_DEF, data, tokens, ts, te)
    };

    T_KEYWORD_END => {
      emit(:T_KEYWORD_END, data, tokens, ts, te)
    };

    T_KEYWORD_OUT => {
      emit(:T_KEYWORD_OUT, data, tokens, ts, te)
    };

    T_DQUOTE => {
      emit(:T_DQUOTE, data, tokens, ts, te)
    };

    T_INT => {
      emit(:T_INT, data, tokens, ts, te)
    };

    T_FLOAT => {
      emit(:T_FLOAT, data, tokens, ts, te)
    };

    T_LITERAL => {
      emit(:T_LITERAL, data, tokens, ts, te)
    };

    T_ADD => {
      emit(:T_ADD, data, tokens, ts, te)
    };

    T_SUB => {
      emit(:T_SUB, data, tokens, ts, te)
    };

    T_MUL => {
      emit(:T_MUL, data, tokens, ts, te)
    };

    T_DIV => {
      emit(:T_DIV, data, tokens, ts, te)
    };

    T_EXP => {
      emit(:T_EXP, data, tokens, ts, te)
    };

    T_MOD => {
      emit(:T_MOD, data, tokens, ts, te)
    };

    T_LBR => {
      emit(:T_LBR, data, tokens, ts, te)
    };

    T_RBR => {
      emit(:T_RBR, data, tokens, ts, te)
    };

    T_EQ => {
      emit(:T_EQ, data, tokens, ts, te)
    };

    T_COL => {
      emit(:T_COL, data, tokens, ts, te)
    };

    space;

  *|;

}%%

module CakeLang
  class Lexer
      CONVERSION = {
          :T_INT => :to_i,
          :T_FLOAT => :to_f
        }

      %% write data;

      # Lazy way of forwarding instance method calls used internally by Ragel to
      # their corresponding class methods.
      private_methods.grep(/^_lexer_/).each do |name|
        define_method(name) do
          return self.class.send(name)
        end

        private(name)
      end

      def emit(symbol, data, tokens, ts, te)
        value = data[ts..te].pack("U*").strip

        if symbol.eql?(:T_INT) or symbol.eql?(:T_FLOAT)
          value = value.send(CONVERSION[symbol])
        end

        tokens << [symbol, value]
      end

      def lex(data)
        data = data.unpack("U*")
        lexer_start = self.class.lexer_start
        eof = data.length
        tokens = []

        %% write init;
        %% write exec;

        tokens
      end
  end
end


Fix SBD rule

Fix SBD rule

%%{

machine sentencize;

alphtype unsigned char;

include whitespace "whitespace.rl";
include misc "misc.rl";
include abbr_lexicon "abbr_lexicon.rl";

#-------------------------------------------------------------------------------
# Character classes.
#-------------------------------------------------------------------------------

eos_marker = "." | "?" | "!";

closing_bracket = ")" | "]" | "}";

closing_quote = closing_single_quote | closing_double_quote;

#-------------------------------------------------------------------------------
# Where to split.
#-------------------------------------------------------------------------------

# Symbols that can appear at the end of a sentence, after the terminal period.
# Only useful for English. In French, these should appear _before_ the
# terminal period.
eos_trail = closing_bracket? closing_single_quote? closing_double_quote?
          | closing_single_quote? closing_double_quote? closing_bracket?
          ;

eos = eos_marker+ eos_trail;

#-------------------------------------------------------------------------------
# Where _not_ to split.
#-------------------------------------------------------------------------------

# Alphanumeric containing internal periods: Ph.D., 1.2.c, e.g., i.e.
thing_with_periods = latin+ ("." latin+)+ "."?;

# A token followed by a period followed by a comma is an abbreviation.
period_comma = latin+ "." eos_trail ",";

# John J. Doe.
name_initial = latin_uppercase "." whitespace latin_uppercase;

# Example:
#    See p. 5 for more informations.
page_number = "p." whitespace digit;

consonant = [BCDFGHJKLMNPQRSTVWXZ] | [bcdfghjklmnpqrstvwxz];

# Example:
#    cf.
abbreviation_consonant = consonant+ ".";

# Interjection:
#    Ah! princesse.
#    "Good gracious!" cried Mrs. Bennet.
# Note that (latin - latin_uppercase) doesn't necessarily match lowercase
# letters.
interjection = latin+ "!" closing_quote? whitespace (latin - latin_uppercase);

not_eos = thing_with_periods
        | period_comma
        | name_initial
        | page_number
        | interjection
        | abbr_lexicon
        | abbreviation_consonant
        | email | uri
        ;

#-------------------------------------------------------------------------------
# Main.
#-------------------------------------------------------------------------------

find_eos := |*
   "." eos_trail => {
      *period = ts;
      goto found;
   };
   eos => {
      goto found;
   };
   paragraph_break => {
      /* Don't include the paragraph break itself in the sentence. */
      te = ts;
      goto found;
   };
   any;
   latin_letter+;
   not_eos;
*|;

main := whitespace* %{ start = fpc; }
        (code_point - whitespace) @{ fexec start; fcall find_eos; };

}%%

%% write data noerror nofinal;

static const unsigned char *next_sentence(struct mr_sentencizer *tkr,
                                          size_t *len,
                                          const unsigned char **period)
{
   int cs, act, top, stack[1];
   const unsigned char *ts, *te;
   const unsigned char *p = tkr->p;
   const unsigned char *pe = tkr->pe;
   const unsigned char *const eof = pe;

   const unsigned char *start = NULL;
   *period = NULL;

   %% write init;
   %% write exec;

   /* Last sentence. Don't know how to trim whitespace on the right. */
   if (start) {
      te = eof;
      goto found;
   }
   return NULL;

found:
   *len = te - start;
   tkr->p = te;
   return start;

   (void)stack;
   (void)sentencize_en_main;
   (void)sentencize_en_find_eos;
}

%%{

machine sentencize;

alphtype unsigned char;

include whitespace "whitespace.rl";
include misc "misc.rl";
include abbr_lexicon "abbr_lexicon.rl";

#-------------------------------------------------------------------------------
# Character classes.
#-------------------------------------------------------------------------------

eos_marker = "." | "?" | "!";

closing_bracket = ")" | "]" | "}";

closing_quote = closing_single_quote | closing_double_quote;

#-------------------------------------------------------------------------------
# Where to split.
#-------------------------------------------------------------------------------

# Symbols that can appear at the end of a sentence, after the terminal period.
# Only useful for English. In French, these should appear _before_ the
# terminal period.
eos_trail = closing_bracket? closing_single_quote? closing_double_quote?
          | closing_single_quote? closing_double_quote? closing_bracket?
          ;

eos = eos_marker+ eos_trail;

#-------------------------------------------------------------------------------
# Where _not_ to split.
#-------------------------------------------------------------------------------

# Alphanumeric containing internal periods: Ph.D., 1.2.c, e.g., i.e.
thing_with_periods = latin+ ("." latin+)+ "."?;

# A token followed by a period followed by a comma is an abbreviation.
period_comma = latin+ "." eos_trail ",";

# John J. Doe.
name_initial = latin_uppercase "." whitespace latin_uppercase;

# Example:
#    See p. 5 for more informations.
page_number = "p." whitespace digit;

# Example:
#    cf.
abbreviation_consonant = [bcdfghjklmnpqrstvwxz]+ ".";

# Interjection:
#    Ah! princesse.
#    "Good gracious!" cried Mrs. Bennet.
# Note that (latin - latin_uppercase) doesn't necessarily match lowercase
# letters.
interjection = latin+ "!" closing_quote? whitespace (latin - latin_uppercase);

not_eos = thing_with_periods
        | period_comma
        | name_initial
        | page_number
        | interjection
        | abbr_lexicon
        | abbreviation_consonant
        | email | uri
        ;

#-------------------------------------------------------------------------------
# Main.
#-------------------------------------------------------------------------------

find_eos := |*
   "." eos_trail => {
      *period = ts;
      goto found;
   };
   eos => {
      goto found;
   };
   paragraph_break => {
      /* Don't include the paragraph break itself in the sentence. */
      te = ts;
      goto found;
   };
   any;
   latin_letter+;
   not_eos;
*|;

main := whitespace* %{ start = fpc; }
        (code_point - whitespace) @{ fexec start; fcall find_eos; };

}%%

%% write data noerror nofinal;

static const unsigned char *next_sentence(struct mr_sentencizer *tkr,
                                          size_t *len,
                                          const unsigned char **period)
{
   int cs, act, top, stack[1];
   const unsigned char *ts, *te;
   const unsigned char *p = tkr->p;
   const unsigned char *pe = tkr->pe;
   const unsigned char *const eof = pe;

   const unsigned char *start = NULL;
   *period = NULL;

   %% write init;
   %% write exec;

   /* Last sentence. Don't know how to trim whitespace on the right. */
   if (start) {
      te = eof;
      goto found;
   }
   return NULL;

found:
   *len = te - start;
   tkr->p = te;
   return start;

   (void)stack;
   (void)sentencize_en_main;
   (void)sentencize_en_find_eos;
}