tokenizing STRING can leave off last double quote

this allows an incomplete STRING to be reported correctly:

    bp(GOBP:"Apop
    IDENT("bp", 0, 2) O_PAREN("(", 2, 3) IDENT("GOBP", 3, 7) COLON(":", 7,
8) STRING(""Apop", 8, 13)

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-token.h"

%%{
    machine set;
    write data;
}%%

bel_token_list* bel_tokenize_term(char* line) {
    int                  cs;
    char                 *p;
    char                 *pe;
    char                 *ts;
    char                 *te;
    char                 *eof;
    int                  token_i;
    bel_token_list       *token_list;
    bel_token            *tokens;
    bel_token            *new_token;

    p                  = line;
    pe                 = line + strlen(line);
    eof                = pe;
    token_i            = 0;
    token_list         = bel_new_token_list(256);
    tokens             = token_list->tokens;

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"'?;

        action IDENT   {
            new_token = bel_new_token(IDENT, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action STRING  {
            new_token = bel_new_token(STRING, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action O_PAREN {
            new_token = bel_new_token(O_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action C_PAREN {
            new_token = bel_new_token(C_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COLON   {
            new_token = bel_new_token(COLON, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COMMA   {
            new_token = bel_new_token(COMMA, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action SPACES  {
            new_token = bel_new_token(SPACES, line, ts, te);
            tokens[token_i++] = *new_token;
        }

        term := |*
            IDENT    => IDENT;
            STRING   => STRING;
            O_PAREN  => O_PAREN;
            C_PAREN  => C_PAREN;
            COLON    => COLON;
            COMMA    => COMMA;
            SPACES   => SPACES;
        *|;

        # Initialize and execute.
        write init;
        write exec;
    }%%

    token_list->length = token_i;
    return token_list;
};
// vim: ft=c sw=4 ts=4 sts=4 expandtab

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "bel-token.h"

%%{
    machine set;
    write data;
}%%

bel_token_list* bel_tokenize_term(char* line) {
    int                  cs;
    char                 *p;
    char                 *pe;
    char                 *ts;
    char                 *te;
    int                  act;
    char                 *eof;
    int                  token_i;
    bel_token_list       *token_list;
    bel_token            *tokens;
    bel_token            *new_token;

    p                  = line;
    pe                 = line + strlen(line);
    eof                = pe;
    token_i            = 0;
    token_list         = bel_new_token_list(256);
    tokens             = token_list->tokens;

    %%{
        C_PAREN      = ')';
        COLON        = ':';
        COMMA        = ',';
        IDENT        = [a-zA-Z0-9_]+;
        O_PAREN      = '(';
        SPACES       = [ ]+;
        STRING       = '"' ('\\\"' | [^"])* '"';

        action IDENT   {
            new_token = bel_new_token(IDENT, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action STRING  {
            new_token = bel_new_token(STRING, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action O_PAREN {
            new_token = bel_new_token(O_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action C_PAREN {
            new_token = bel_new_token(C_PAREN, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COLON   {
            new_token = bel_new_token(COLON, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action COMMA   {
            new_token = bel_new_token(COMMA, line, ts, te);
            tokens[token_i++] = *new_token;
        }
        action SPACES  {
            new_token = bel_new_token(SPACES, line, ts, te);
            tokens[token_i++] = *new_token;
        }

        term := |*
            IDENT    => IDENT;
            STRING   => STRING;
            O_PAREN  => O_PAREN;
            C_PAREN  => C_PAREN;
            COLON    => COLON;
            COMMA    => COMMA;
            SPACES   => SPACES;
        *|;

        # Initialize and execute.
        write init;
        write exec;
    }%%

    token_list->length = token_i;
    return token_list;
};
// vim: ft=c sw=4 ts=4 sts=4 expandtab


Dump finite state machine data in anon namespace.

// Experimental parser for macros
//
// TODO:
//
// - Eat whitespace between opening/closing quotes and the actual query.
// - Add error handling.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <sstream>
#include <vector>

#include <cstring>

namespace {

%%{
	machine macros;
	write data;
}%%

struct Substitution
{
	size_t begin;
	size_t n;
	std::string macro;
};

}

typedef std::map<std::string, std::string> Macros;
typedef std::vector<Substitution> Substitutions;

Macros parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;

	char const *substStart = 0;

	Macros macros;

	Substitutions substitutions;

	std::string lastKey;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		lastKey = buf.str();
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		// This action is executed when the query is closed ("""), we have to
		// three to avoid including the quotes.
		std::string query(strStart, p - 3);

		// Apply substitutions that were found in the macro.
		for (Substitutions::const_reverse_iterator iter = substitutions.rbegin();
			iter != substitutions.rend(); ++iter)
		{
			Macros::const_iterator mIter = macros.find(iter->macro);
			if (mIter == macros.end())
			{
				std::cerr << "Unknown macro: " << iter->macro << std::endl;
				continue;
			}

			query.replace(iter->begin, iter->n, mIter->second);
		}

		substitutions.clear();
		macros[lastKey] = query;
	}

	action substBegin {
		substStart = p;
	}

	action substEnd {
		std::string macro(substStart, p);

		// substStart is the position of the macro name. Decrement by one to
		// get the position of the percentage sign.
		size_t begin = substStart - strStart - 1;

		// Add two, to account for both percentage signs.
		size_t n = p - substStart + 2;

		Substitution subst = {begin, n, macro};
		substitutions.push_back(subst);
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	substitution = ('%' % substBegin) (any - '%')+ ('%' > substEnd);
	query = (substitution | (any - '%')+)* -- separator;
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%

	return macros;
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();
	//std::string test = "\"\"\"test\"\"\"";

	//parseMacros(test.c_str());

	Macros macros = parseMacros(data.c_str());
	for (Macros::const_iterator iter = macros.begin(); iter != macros.end(); ++iter)
		std::cout << iter->first << ": " << iter->second << std::endl;
}

// Experimental parser for macros
//
// TODO:
//
// - Eat whitespace between opening/closing quotes and the actual query.
// - Add error handling.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <sstream>
#include <vector>

#include <cstring>

%%{
	machine macros;
	write data;
}%%

namespace {

struct Substitution
{
	size_t begin;
	size_t n;
	std::string macro;
};

}

typedef std::map<std::string, std::string> Macros;
typedef std::vector<Substitution> Substitutions;

Macros parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;

	char const *substStart = 0;

	Macros macros;

	Substitutions substitutions;

	std::string lastKey;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		lastKey = buf.str();
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		// This action is executed when the query is closed ("""), we have to
		// three to avoid including the quotes.
		std::string query(strStart, p - 3);

		// Apply substitutions that were found in the macro.
		for (Substitutions::const_reverse_iterator iter = substitutions.rbegin();
			iter != substitutions.rend(); ++iter)
		{
			Macros::const_iterator mIter = macros.find(iter->macro);
			if (mIter == macros.end())
			{
				std::cerr << "Unknown macro: " << iter->macro << std::endl;
				continue;
			}

			query.replace(iter->begin, iter->n, mIter->second);
		}

		substitutions.clear();
		macros[lastKey] = query;
	}

	action substBegin {
		substStart = p;
	}

	action substEnd {
		std::string macro(substStart, p);

		// substStart is the position of the macro name. Decrement by one to
		// get the position of the percentage sign.
		size_t begin = substStart - strStart - 1;

		// Add two, to account for both percentage signs.
		size_t n = p - substStart + 2;

		Substitution subst = {begin, n, macro};
		substitutions.push_back(subst);
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	substitution = ('%' % substBegin) (any - '%')+ ('%' > substEnd);
	query = (substitution | (any - '%')+)* -- separator;
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%

	return macros;
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();
	//std::string test = "\"\"\"test\"\"\"";

	//parseMacros(test.c_str());

	Macros macros = parseMacros(data.c_str());
	for (Macros::const_iterator iter = macros.begin(); iter != macros.end(); ++iter)
		std::cout << iter->first << ": " << iter->second << std::endl;
}