Interpolation?

Interpolation?

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( >mark number         %{ emit :number     } ) |
           ( >mark string_double  %{ emit :dstring    } ) |
           ( >mark string_single  %{ emit :sstring    } ) |
           ( >mark istring_start  %{ emit :istart     } ) |
           ( >mark istring_middle %{ emit :imiddle    } ) |
           ( >mark istring_end    %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( >mark unops          %{ emit :unop       } ) |
           ( >mark binops         %{ emit :binop      } ) |
           ( >mark keywords       %{ emit :keyword    } ) |
           ( >mark identifier     %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' ( any -- '"' | '\\"' )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  body = ( >mark number        %{ emit :number     } ) |
         ( >mark string_double %{ emit :dstring    } ) |
         ( >mark string_single %{ emit :sstring    } ) |
         (       'class'       %{ emit :class      } ) |
         (       'module'      %{ emit :module     } ) |
         (       'if'          %{ emit :if         } ) |
         (       'unless'      %{ emit :unless     } ) |
         (       'elsif'       %{ emit :elsif      } ) |
         (       'else'        %{ emit :else       } ) |
         ( >mark unops         %{ emit :unop       } ) |
         ( >mark binops        %{ emit :binop      } ) |
         ( >mark keywords      %{ emit :keyword    } ) |
         ( >mark identifier    %{ emit :identifier } ) |
         (       '->'          %{ emit :arrow      } ) |
         (       '='           %{ emit :equal      } ) |
         (       '{'           %{ emit :lbrack     } ) |
         (       '('           %{ emit :lparen     } ) |
         (       '['           %{ emit :lbrace     } ) |
         (       '}'           %{ emit :rbrack     } ) |
         (       ')'           %{ emit :rparen     } ) |
         (       ']'           %{ emit :rbrace     } ) |
         (       ':'           %{ emit :colon      } ) |
         (       '.'           %{ emit :prop       } ) |
         (       ','           %{ emit :comma      } ) |
         (       '\n'          %{ line.call        } )
  ;

  main := body**;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type, cur = nil)
        @tokens << if cur
          Token.new(type, @data[@start..(@cur - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@ts..(@te-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end


Remove admin.rl inherited with merge.

Remove admin.rl inherited with merge.

/*
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * 1. Redistributions of source code must retain the above
 *    copyright notice, this list of conditions and the
 *    following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above
 *    copyright notice, this list of conditions and the following
 *    disclaimer in the documentation and/or other materials
 *    provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
 * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include <stdio.h>
#include <string.h>
#include <stdbool.h>
#include <unistd.h>
#include <stdlib.h>

#include <fiber.h>
#include <palloc.h>
#include <salloc.h>
#include <say.h>
#include <stat.h>
#include <tarantool.h>
#include "lua/init.h"
#include <recovery.h>
#include <tbuf.h>
#include "tarantool/util.h"
#include <errinj.h>
#include "coio_buf.h"

extern "C" {
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
}

#include "box/box.h"
#include "lua/init.h"
#include "session.h"
#include "scoped_guard.h"

static const char *help =
	"available commands:" CRLF
	" - help" CRLF
	" - exit" CRLF
	" - show info" CRLF
	" - show fiber" CRLF
	" - show configuration" CRLF
	" - show slab" CRLF
	" - show palloc" CRLF
	" - show stat" CRLF
	" - show plugins" CRLF
	" - save coredump" CRLF
	" - save snapshot" CRLF
	" - lua command" CRLF
	" - reload configuration" CRLF
	" - show injections (debug mode only)" CRLF
	" - set injection <name> <state> (debug mode only)" CRLF;

static const char *unknown_command = "unknown command. try typing help." CRLF;

%%{
	machine admin;
	write data;
}%%

struct salloc_stat_admin_cb_ctx {
	int64_t total_used;
	struct tbuf *out;
};

static int
salloc_stat_admin_cb(const struct slab_cache_stats *cstat, void *cb_ctx)
{
	struct salloc_stat_admin_cb_ctx *ctx = (struct salloc_stat_admin_cb_ctx *) cb_ctx;

	tbuf_printf(ctx->out,
		    "     - { item_size: %- 5i, slabs: %- 3i, items: %- 11" PRIi64
		    ", bytes_used: %- 12" PRIi64
		    ", bytes_free: %- 12" PRIi64 " }" CRLF,
		    (int)cstat->item_size,
		    (int)cstat->slabs,
		    cstat->items,
		    cstat->bytes_used,
		    cstat->bytes_free);

	ctx->total_used += cstat->bytes_used;
	return 0;
}

static void
show_slab(struct tbuf *out)
{
	struct salloc_stat_admin_cb_ctx cb_ctx;
	struct slab_arena_stats astat;

	cb_ctx.total_used = 0;
	cb_ctx.out = out;

	tbuf_printf(out, "slab statistics:\n  classes:" CRLF);

	salloc_stat(salloc_stat_admin_cb, &astat, &cb_ctx);

	tbuf_printf(out, "  items_used: %.2f%%" CRLF,
		(double)cb_ctx.total_used / astat.size * 100);
	tbuf_printf(out, "  arena_used: %.2f%%" CRLF,
		(double)astat.used / astat.size * 100);
}

static void
end(struct tbuf *out)
{
	tbuf_printf(out, "..." CRLF);
}

static void
start(struct tbuf *out)
{
	tbuf_printf(out, "---" CRLF);
}

static void
ok(struct tbuf *out)
{
	start(out);
	tbuf_printf(out, "ok" CRLF);
	end(out);
}

static void
fail(struct tbuf *out, struct tbuf *err)
{
	start(out);
	tbuf_printf(out, "fail:%.*s" CRLF, err->size, (char *)err->data);
	end(out);
}

static void
tarantool_info(struct tbuf *out)
{
	tbuf_printf(out, "info:" CRLF);
	tbuf_printf(out, "  version: \"%s\"" CRLF, tarantool_version());
	tbuf_printf(out, "  uptime: %i" CRLF, (int)tarantool_uptime());
	tbuf_printf(out, "  pid: %i" CRLF, getpid());
	tbuf_printf(out, "  logger_pid: %i" CRLF, logger_pid);
	tbuf_printf(out, "  snapshot_pid: %i" CRLF, snapshot_pid);
	tbuf_printf(out, "  lsn: %" PRIi64 CRLF,
		    recovery_state->confirmed_lsn);
	tbuf_printf(out, "  recovery_lag: %.3f" CRLF,
		    recovery_state->remote ?
		    recovery_state->remote->recovery_lag : 0);
	tbuf_printf(out, "  recovery_last_update: %.3f" CRLF,
		    recovery_state->remote ?
		    recovery_state->remote->recovery_last_update_tstamp :0);
	box_info(out);
	const char *path = cfg_filename_fullpath;
	if (path == NULL)
		path = cfg_filename;
	tbuf_printf(out, "  config: \"%s\"" CRLF, path);
}

static int
show_stat_item(const char *name, int rps, int64_t total, void *ctx)
{
	struct tbuf *buf = (struct tbuf *) ctx;
	int name_len = strlen(name);
	tbuf_printf(buf,
		    "  %s:%*s{ rps: %- 6i, total: %- 12" PRIi64 " }" CRLF,
		    name, 1 + stat_max_name_len - name_len, " ", rps, total);
	return 0;
}

void
show_stat(struct tbuf *buf)
{
	tbuf_printf(buf, "statistics:" CRLF);
	stat_foreach(show_stat_item, buf);
}

static int
admin_dispatch(struct ev_io *coio, struct iobuf *iobuf, lua_State *L)
{
	struct ibuf *in = &iobuf->in;
	struct tbuf *out = tbuf_new(fiber->gc_pool);
	struct tbuf *err = tbuf_new(fiber->gc_pool);
	int cs;
	char *p, *pe;
	char *strstart, *strend;
	bool state;

	while ((pe = (char *) memchr(in->pos, '\n', in->end - in->pos)) == NULL) {
		if (coio_bread(coio, in, 1) <= 0)
			return -1;
	}

	pe++;
	p = in->pos;

	%%{
	        action show_plugins {
		    start(out);
                    show_plugins_stat(out);
		    end(out);
                }

		action show_configuration {
			start(out);
			show_cfg(out);
			end(out);
		}

		action show_injections {
			start(out);
			errinj_info(out);
			end(out);
		}

		action help {
			start(out);
			tbuf_append(out, help, strlen(help));
			end(out);
		}

		action lua {
			strstart[strend-strstart]='\0';
			start(out);
			tarantool_lua(L, out, strstart);
			end(out);
		}

		action reload_configuration {
			if (reload_cfg(err))
				fail(out, err);
			else
				ok(out);
		}

		action save_snapshot {
			int ret = snapshot();

			if (ret == 0)
				ok(out);
			else {
				tbuf_printf(err, " can't save snapshot, errno %d (%s)",
					    ret, strerror(ret));

				fail(out, err);
			}
		}

		action set_injection {
			strstart[strend-strstart] = '\0';
			if (errinj_set_byname(strstart, state)) {
				tbuf_printf(err, "can't find error injection '%s'", strstart);
				fail(out, err);
			} else {
				ok(out);
			}
		}

		eol = "\n" | "\r\n";
		show = "sh"("o"("w")?)?;
		info = "in"("f"("o")?)?;
		check = "ch"("e"("c"("k")?)?)?;
		configuration = "co"("n"("f"("i"("g"("u"("r"("a"("t"("i"("o"("n")?)?)?)?)?)?)?)?)?)?)?;
		fiber = "fi"("b"("e"("r")?)?)?;
		slab = "sl"("a"("b")?)?;
		mod = "mo"("d")?;
		palloc = "pa"("l"("l"("o"("c")?)?)?)?;
		stat = "st"("a"("t")?)?;
		plugins = "plugins";

		help = "h"("e"("l"("p")?)?)?;
		exit = "e"("x"("i"("t")?)?)? | "q"("u"("i"("t")?)?)?;
		save = "sa"("v"("e")?)?;
		coredump = "co"("r"("e"("d"("u"("m"("p")?)?)?)?)?)?;
		snapshot = "sn"("a"("p"("s"("h"("o"("t")?)?)?)?)?)?;
		string = [^\r\n]+ >{strstart = p;}  %{strend = p;};
		reload = "re"("l"("o"("a"("d")?)?)?)?;
		lua = "lu"("a")?;

		set = "se"("t")?;
		injection = "in"("j"("e"("c"("t"("i"("o"("n")?)?)?)?)?)?)?;
		injections = injection"s";
		namech = alnum | punct;
		name = namech+ >{ strstart = p; }  %{ strend = p; };
		state_on = "on" %{ state = true; };
		state_off = "of"("f")? %{ state = false; };
		state = state_on | state_off;

		commands = (help			%help						|
			    exit			%{return -1;}					|
			    lua  " "+ string		%lua						|
			    show " "+ info		%{start(out); tarantool_info(out); end(out);}	|
			    show " "+ fiber		%{start(out); fiber_info(out); end(out);}	|
			    show " "+ configuration 	%show_configuration				|
			    show " "+ slab		%{start(out); show_slab(out); end(out);}	|
			    show " "+ palloc		%{start(out); palloc_stat(out); end(out);}	|
			    show " "+ stat		%{start(out); show_stat(out);end(out);}		|
			    show " "+ injections	%show_injections                                |
			    show " "+ plugins           %show_plugins                                   |
			    set " "+ injection " "+ name " "+ state	%set_injection                  |
			    save " "+ coredump		%{coredump(60); ok(out);}			|
			    save " "+ snapshot		%save_snapshot					|
			    check " "+ slab		%{slab_validate(); ok(out);}			|
			    reload " "+ configuration	%reload_configuration);

		main := commands eol;
		write init;
		write exec;
	}%%

	in->pos = pe;

	if (p != pe) {
		start(out);
		tbuf_append(out, unknown_command, strlen(unknown_command));
		end(out);
	}

	coio_write(coio, out->data, out->size);
	return 0;
}

static void
admin_handler(va_list ap)
{
	struct ev_io coio = va_arg(ap, struct ev_io);
	struct sockaddr_in *addr = va_arg(ap, struct sockaddr_in *);
	struct iobuf *iobuf = va_arg(ap, struct iobuf *);
	lua_State *L = lua_newthread(tarantool_L);
	int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX);

	auto scoped_guard = make_scoped_guard([&] {
		luaL_unref(tarantool_L, LUA_REGISTRYINDEX, coro_ref);
		evio_close(&coio);
		iobuf_delete(iobuf);
		session_destroy(fiber->sid);
	});

	/*
	 * Admin and iproto connections must have a
	 * session object, representing the state of
	 * a remote client: it's used in Lua
	 * stored procedures.
	 */
	session_create(coio.fd, *(uint64_t *) addr);
	for (;;) {
		if (admin_dispatch(&coio, iobuf, L) < 0)
			return;
		iobuf_gc(iobuf);
		fiber_gc();
	}
}

void
admin_init(const char *bind_ipaddr, int admin_port)
{
	static struct coio_service admin;
	coio_service_init(&admin, "admin", bind_ipaddr,
			  admin_port, admin_handler, NULL);
	evio_service_start(&admin.evio_service);
}

/*
 * Local Variables:
 * mode: c
 * End:
 * vim: syntax=objc
 */


Parse all tag types

Parse all tag types

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = [ \t]*;
	open  = '{' :> '{';
	close = '}' :> '}';

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/_] | '-')+;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
		printf("Start of %c tag\n", tag_type);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
	}

	action set_type {
		tag_type = *fpc;
	}

	action init_type {
		tag_type = '\0';
	}

	type = [#^/=!<>&];

	var = (
		open
		type? >init_type $set_type <:
		white
		identifier >start_identifier %got_identifier
		white
		close %mark
	) >write_static;

	# Special case for triple mustache
	unescaped = (
		open
		'{'
		white
		identifier >start_identifier %got_identifier
		white
		'}'
		close %mark
	) >write_static;

	text = (any+ -- open) ;

	body = (
		var
		| text
		| unescaped
	);

	main := body* %eof(write_static);

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;
  char tag_type = '\0';

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = [ \t]*;
	open  = '{' :> '{';
	close = '}' :> '}';

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/] | '-')+;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
	}

	#tag_type = [&/<>{^];

	tag = (
		open
		white
		identifier >start_identifier %got_identifier
		white
		close %mark
	) >write_static;

	text = (any+ -- open) ;

	main := (
		tag |
		text
	)* %eof(write_static);

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end


Avoid leaking the strdup'd segments pushed to the queue.

/* -*-c-*- */
/* This file is part of liberis
 *
 * Copyright (C) 2008 by LScube team <team@streaming.polito.it>
 * See AUTHORS for more details
 *
 * liberis is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * liberis is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with liberis.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include <config.h>
#include <string.h>
#include "uri.h"

%%{
    machine uri_parser;
	include common "common.rl";

    action collapse_path {
        if ( ! g_queue_is_empty(stack) ) {
            GString *uri_path = g_string_new("");
            g_queue_foreach(stack, append_to_string, uri_path);
            uri->path = g_string_free(uri_path, FALSE);
        }
    }

	reg_name = ( unreserved | pct_encoded | sub_delims )*;
	host = IP_literal | IPv4address | reg_name;
	authority = ( userinfo >mark
                     %{uri->userinfo = g_strndup(mark, fpc-mark);}
                     "@" )?
                    host >mark
                     %{uri->host = g_strndup(mark, fpc-mark);}
                    ( ":" digit* >mark
                     %{uri->port = g_strndup(mark, fpc-mark);})?;
	pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
	segment = ( pchar+ - ".." - "." ) >mark
                        %{mark = g_strndup(mark, fpc-mark);
                          g_queue_push_tail(stack, mark);}|
                  ".." %{g_queue_pop_tail(stack);}|
                    zlen | ".";
	path_abempty = ( "/" segment )*;
	segment_nz = ( pchar+ - ".." ) | "..";
	path_absolute = "/" ( segment_nz ( "/" segment )* )?;
	path_rootless = segment_nz ( "/" segment )*;
	path_empty = empty;
	hier_part = ( "//" authority path_abempty ) | path_absolute | path_rootless | path_empty;
	query = ( pchar | "/" | "?" )*;
	fragment = ( pchar | "/" | "?" )*;
	URI = scheme >mark
               %{uri->scheme = g_strndup(mark, fpc-mark);}
              ":" hier_part ( "?" query >mark
               %{uri->query = g_strndup(mark, fpc-mark);}
                )? ( "#" fragment >mark
               %{uri->fragment = g_strndup(mark, fpc-mark);} )?
               %(collapse_path);
	segment_nz_nc = ( unreserved | pct_encoded | sub_delims | "@" )+;
	path_noscheme = segment_nz_nc ( "/" segment )*;
	relative_part = ( "//" authority path_abempty ) | path_absolute | path_noscheme | path_empty;
	relative_ref = relative_part ( "?" query )? ( "#" fragment )?;
	URI_reference = URI | relative_ref;
	absolute_URI = scheme ":" hier_part ( "?" query )?;
	path = path_abempty | path_absolute | path_noscheme | path_rootless | path_empty;
	gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
	reserved = gen_delims | sub_delims;

    # instantiate machine rules
    main:= URI 0;
}%%

static void append_to_string(gpointer data, gpointer user_data)
{
    GString *path = (GString*)user_data;
    char *segment = (char *)data;
    g_string_append_printf(path, "/%s", segment);
}

static void segment_free(gpointer data, ATTR_UNUSED gpointer unused)
{
    g_free(data);
}

/**
 * @brief Parse and split an URI string into an URI structure
 *
 * @param uri_string The URI string to parse and split
 *
 * @return A pointer to a newly-allocated URI structure.
 *
 * This function parses an URI, as defined by RFC1738, and splits it
 * into its components in the given structure.
 */
URI *uri_parse(const char *uri_string)
{
    const char *p = uri_string, *pe = p + strlen(p) + 1, *mark;
    GQueue *stack = g_queue_new();
    URI *uri = g_slice_new0(URI);
    int cs;

    %% write data;
    %% write init;
    %% write exec;

    g_queue_foreach(stack, segment_free, NULL);
    g_queue_free(stack);

    if (cs < uri_parser_first_final) {
        uri_free(uri);
        return NULL;
    }

    return uri;
}

/* -*-c-*- */
/* This file is part of liberis
 *
 * Copyright (C) 2008 by LScube team <team@streaming.polito.it>
 * See AUTHORS for more details
 *
 * liberis is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * liberis is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with liberis.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include <config.h>
#include <string.h>
#include "uri.h"

%%{
    machine uri_parser;
	include common "common.rl";

    action collapse_path {
        if ( ! g_queue_is_empty(stack) ) {
            GString *uri_path = g_string_new("");
            g_queue_foreach(stack, append_to_string, uri_path);
            uri->path = g_string_free(uri_path, FALSE);
        }
    }

	reg_name = ( unreserved | pct_encoded | sub_delims )*;
	host = IP_literal | IPv4address | reg_name;
	authority = ( userinfo >mark
                     %{uri->userinfo = g_strndup(mark, fpc-mark);}
                     "@" )?
                    host >mark
                     %{uri->host = g_strndup(mark, fpc-mark);}
                    ( ":" digit* >mark
                     %{uri->port = g_strndup(mark, fpc-mark);})?;
	pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
	segment = ( pchar+ - ".." - "." ) >mark
                        %{mark = g_strndup(mark, fpc-mark);
                          g_queue_push_tail(stack, mark);}|
                  ".." %{g_queue_pop_tail(stack);}|
                    zlen | ".";
	path_abempty = ( "/" segment )*;
	segment_nz = ( pchar+ - ".." ) | "..";
	path_absolute = "/" ( segment_nz ( "/" segment )* )?;
	path_rootless = segment_nz ( "/" segment )*;
	path_empty = empty;
	hier_part = ( "//" authority path_abempty ) | path_absolute | path_rootless | path_empty;
	query = ( pchar | "/" | "?" )*;
	fragment = ( pchar | "/" | "?" )*;
	URI = scheme >mark
               %{uri->scheme = g_strndup(mark, fpc-mark);}
              ":" hier_part ( "?" query >mark
               %{uri->query = g_strndup(mark, fpc-mark);}
                )? ( "#" fragment >mark
               %{uri->fragment = g_strndup(mark, fpc-mark);} )?
               %(collapse_path);
	segment_nz_nc = ( unreserved | pct_encoded | sub_delims | "@" )+;
	path_noscheme = segment_nz_nc ( "/" segment )*;
	relative_part = ( "//" authority path_abempty ) | path_absolute | path_noscheme | path_empty;
	relative_ref = relative_part ( "?" query )? ( "#" fragment )?;
	URI_reference = URI | relative_ref;
	absolute_URI = scheme ":" hier_part ( "?" query )?;
	path = path_abempty | path_absolute | path_noscheme | path_rootless | path_empty;
	gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
	reserved = gen_delims | sub_delims;

    # instantiate machine rules
    main:= URI 0;
}%%

static void append_to_string(gpointer data, gpointer user_data)
{
    GString *path = (GString*)user_data;
    char *segment = (char *)data;
    g_string_append_printf(path, "/%s", segment);
}

/**
 * @brief Parse and split an URI string into an URI structure
 *
 * @param uri_string The URI string to parse and split
 *
 * @return A pointer to a newly-allocated URI structure.
 *
 * This function parses an URI, as defined by RFC1738, and splits it
 * into its components in the given structure.
 */
URI *uri_parse(const char *uri_string)
{
    const char *p = uri_string, *pe = p + strlen(p) + 1, *mark;
    GQueue *stack = g_queue_new();
    URI *uri = g_slice_new0(URI);
    int cs;

    %% write data;
    %% write init;
    %% write exec;

    g_queue_free(stack);

    if (cs < uri_parser_first_final) {
        uri_free(uri);
        return NULL;
    }

    return uri;
}


went on commenting mtoc

went on commenting mtoc

Conflicts:

	3rd_party/mtoc/confscanner.rl

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string ist matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // check wether str matches glob
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename ist matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

// constructor
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "doxygen/mtoc.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "Error opening configuration file '" << conffilename << "'\n";
    exit(-2);
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    // string for temporary tokens
    string tmp_string;
    // spare space in buffer
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    // read configuration file chunk in buffer
    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  # store current char pointer p in tmp_p
  action st_tok { tmp_p = p; }

  # assign a string token to tmp_string variable
  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  # add groupname to groupset_ if glob expression matches
  action group_add
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  # set cblock_ to selected documentation block as specified in an expression
  # "add(specifier)"
  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  # set clist_ to selected documentation block list as specified in an
  # expression "add(params)" or "add(return)"
  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  # set clistmap_ to selected documentation block list map as specified in an
  # expression "add(fields)"
  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    // check wether glob expression matches
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  # parameter values for single documentation blocks
  add_param_docu_block = ('brief'|'doc'|'extra');
  # parameter values for lists of documentation blocks
  add_param_docu_list = ('params'|'return');
  # parameter values for a map of lists of documentation blocks
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # line in documentation block
  docu_block_line :=
    (
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     EOL @(add_block_line) @{line++;}
     |
     '"""' @(add_block_line) @{fret;}
     |
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a list of documentation block lists
  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  # documentation list map item
  docu_list_map_item =
    (
     # matlab identifier (structure)
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     # matlab identifer (fieldname)
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  # a map of documentation block lists
  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  # list of file matching words
  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  # list of group names
  grouplist = (
     # matlab identifier (group name)
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
     # matlab identifier (group name)
    .(MIDENT >(st_tok)
      %(group_add))
    );

  # specify all possible rules
  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down
    ('}') @{ go_level_down(); fret; }
    )*;

  main := (
           MWSOC
           | (
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

// this method is called when a glob ... {} block ends
void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

// checks wether the string ist matched by a glob from the globlist_stack_ at
// level \a l
bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  // get globlist at stack level \a l
  GlobList & gl = globlist_stack_[l];

  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // iterate over all globs
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // check wether str matches glob
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      return true;
    }
  }
  return false;
}

// check recursively wether file named \a s is matched by a glob from the
// globlist_stack_
bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  // exit condition (if filename ist matched up to level level_+1 the check was
  // successful.
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
    // first try to match the prepended path by a glob at this level, ...
    if(check_for_match(l, str.c_str())
        // ... then try to match the rest of the substring at a higher level.
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string at this level
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

// this method is called after a glob ... = {} block begins
// and sets the match_at_level_ structure for the active file named
// \a filename_
void ConfFileScanner :: check_glob_level_up()
{
  // update match_at_level_
  match_at_level_[level_+1] = check_glob_rec(0, filename_);

  // add empty glob list
  globlist_stack_.push_back(GlobList());
  level_++;
}

// constructor
ConfFileScanner
:: ConfFileScanner(const std::string & filename, const std::string & conffilename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   conffile_(conffilename == "" ? "doxygen/mtoc.conf" : conffilename),
   confistream_(get_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  if ( (confistream_.rdstate() & ifstream::failbit ) != 0 )
  {
    cerr << "Error opening configuration file '" << conffilename << "'\n";
    exit(-2);
  }
  globlist_stack_.push_back(GlobList());
  // at level 0 every file is matched ( no glob checking )
  match_at_level_[0] = true;
}

// returns the name of the configuration file
const char * ConfFileScanner ::
get_conffile()
{
  return conffile_.c_str();
}

// run the conffile scanner
int ConfFileScanner :: execute()
{
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    // string for temporary tokens
    string tmp_string;
    // spare space in buffer
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    // read configuration file chunk in buffer
    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */


Fix leading/trailing whitespace for JSON parser

Operator precedence meant it was either leading whitespace + an object, or an
array + trailing whitespace, not leading + (object or array) + trailing.
Just let it be any type of value.

Change-Id: I537d2cceee53a75c1ef79c36337e7dc655b028ef
Reviewed-on: https://gerrit.dechocorp.com/6346
Reviewed-by: Jeremy Stanley <8cc48e55af0ea00a29d3ccf3190022dccfb4961b@decho.com>

// Copyright (c) 2009 - Mozy, Inc.

#include "mordor/pch.h"

#include "mordor/json.h"

#include "mordor/assert.h"

namespace Mordor {
namespace JSON {

std::string unquote(const std::string &string)
{
    MORDOR_ASSERT(string.size() >= 2);
    MORDOR_ASSERT(string[0] == '"');
    MORDOR_ASSERT(string[string.size() - 1] == '"');
    std::string result = string.substr(1, string.size() - 2);

    const char *c = string.c_str() + 1;
    const char *end = c + string.size() - 2;
    bool differed = false;
    int utf16;
    while (c < end)
    {
        if (*c == '\\') {
            MORDOR_ASSERT(c + 1 < end);
            if (!differed) {
                result.resize(c - (string.c_str() + 1));
                differed = true;
            }
            ++c;
            switch (*c) {
                case '"':
                case '\\':
                case '/':
                    result.append(1, *c);
                    break;
                case 'b':
                    result.append(1, '\b');
                    break;
                case 'f':
                    result.append(1, '\f');
                    break;
                case 'n':
                    result.append(1, '\n');
                    break;
                case 'r':
                    result.append(1, '\r');
                    break;
                case 't':
                    result.append(1, '\t');
                    break;
                case 'u':
                    MORDOR_ASSERT(c + 4 < end);
                    utf16 = 0;
                    ++c;
                    for (int i = 0; i < 4; ++i) {
                        utf16 *= 16;
                        if (*c >= '0' && *c <= '9')
                            utf16 += *c - '0';
                        else if (*c >= 'a' && *c <= 'f')
                            utf16 += *c - 'a' + 10;
                        else if (*c >= 'A' && *c <= 'F')
                            utf16 += *c - 'A' + 10;
                        else
                            MORDOR_NOTREACHED();
                        ++c;
                    }
                    // Utf16->Utf8 conversion not supported yet
                    MORDOR_ASSERT(utf16 < 255);
                    result.append(1, (char)utf16);
                    break;
                default:
                    MORDOR_NOTREACHED();
            }
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

%%{
    machine json_parser;

    action mark { mark = fpc;}
    action done { fbreak; }
    prepush {
        prepush();
    }
    postpop {
        postpop();
    }

    ws = ' ' | '\t' | '\r' | '\n';

    unescaped = (any - ('"' | '\\') - cntrl);
    char = unescaped | ('\\' ('"' | '\\' | 'b' | 'f' | 'n' | 'r' | 't' | ('u' [0-9A-Za-z]{4})));
    string = '"' char* '"';

    action begin_number
    {
        m_nonIntegral = false;
    }
    action set_non_integral
    {
        m_nonIntegral = true;
    }
    action parse_number
    {
        if (m_nonIntegral) {
            *m_stack.top() = strtod(mark, NULL);
        } else {
            *m_stack.top() = strtoll(mark, NULL, 10);
        }
    }

    int = (digit | ([1-9] digit*));
    frac = '.' >set_non_integral digit+;
    exp = 'e'i >set_non_integral ('+' | '-')? digit+;
    number = ('-'? int frac? exp?) >mark >begin_number %parse_number;

    action parse_string
    {
        *m_stack.top() = unquote(std::string(mark, fpc - mark));
    }
    action call_parse_object
    {
        *m_stack.top() = Object();
        fcall *json_parser_en_parse_object;
    }
    action call_parse_array
    {
        *m_stack.top() = Array();
        fcall *json_parser_en_parse_array;
    }
    action parse_true
    {
        *m_stack.top() = true;
    }
    action parse_false
    {
        *m_stack.top() = false;
    }
    action parse_null
    {
        *m_stack.top() = boost::blank();
    }
    value = string >mark %parse_string | number | '{' @call_parse_object |
        '[' @call_parse_array | 'true' @parse_true | 'false' @parse_false |
        'null' @parse_null;

    object = '{' ws* string ws* ':' ws* value ws* (',' ws* string ws* ':' ws* value ws*)* '}';
    array = '[' ws* value ws* (',' ws* value ws*)* ']';

    action new_key
    {
        m_stack.push(&boost::get<Object>(*m_stack.top()).insert(std::make_pair(unquote(std::string(mark, fpc - mark)), Value()))->second);
    }
    action new_element
    {
        boost::get<Array>(*m_stack.top()).push_back(Value());
        m_stack.push(&boost::get<Array>(*m_stack.top()).back());
    }
    action pop_stack
    {
        m_stack.pop();
    }
    action ret {
        fret;
    }

    parse_object := parse_object_lbl: ws* string >mark %new_key ws* ':' ws* value %pop_stack ws* (',' ws* string >mark %new_key ws* ':' ws* value %pop_stack ws*)* '}' @ret;
    parse_array := parse_array_lbl: ws* value >new_element %pop_stack ws* (',' ws* value >new_element %pop_stack ws*)* ']' @ret;

    main := ws* value ws*;
    write data;
}%%

void
Parser::init()
{
    while (m_stack.size() > 1)
        m_stack.pop();
    *m_stack.top() = boost::blank();
    RagelParserWithStack::init();
    %% write init;
}

void
Parser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

bool
Parser::final() const
{
    return cs >= json_parser_first_final;
}

bool
Parser::error() const
{
    return cs == json_parser_error;
}

std::string
quote(const std::string &str)
{
    std::string result;
    result.reserve(str.length() + 2);
    result.append(1, '"');

    static const char *escaped =
        "\0\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
        "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
        "\\\"";

    size_t lastEscape = 0;
    size_t nextEscape = str.find_first_of(escaped, 0, 34);
    std::ostringstream os;
    os.fill('0');
    os << std::hex << std::setw(4);
    while (nextEscape != std::string::npos) {
        result.append(str.substr(lastEscape, nextEscape - lastEscape));
        result.append(1, '\\');
        switch (str[nextEscape]) {
            case '"':
            case '\\':
                result.append(1, str[nextEscape]);
                break;
            case '\b':
                result.append(1, 'b');
                break;
            case '\f':
                result.append(1, 'f');
                break;
            case '\n':
                result.append(1, 'n');
                break;
            case '\r':
                result.append(1, 'r');
                break;
            case '\t':
                result.append(1, 't');
                break;
            default:
                result.append(1, 'u');
                os.str();
                os << (int)str[nextEscape];
                result.append(os.str());
                break;
        }
        lastEscape = nextEscape + 1;
        nextEscape = str.find_first_of(escaped, lastEscape, 34);
    }
    result.append(str.substr(lastEscape));
    result.append(1, '"');
    return result;
}

namespace {
class JSONVisitor : public boost::static_visitor<>
{
public:
    JSONVisitor(std::ostream &os)
    : os(os),
      depth(0)
    {}

    void operator()(const boost::blank &b)
    {
        os << "null";
    }
    void operator()(const bool &b)
    {
        os << (b ? "true" : "false");
    }
    template <class T>
    void operator()(const T &t)
    {
        os << t;
    }
    void operator()(const std::string &str)
    {
        os << quote(str);
    }
    void operator()(const Object &object)
    {
        if (object.empty()) {
            os << "{ }";
        } else {
            ++depth;
            std::string prefix(depth * 4, ' ');
            os << "{\n";
            for (Object::const_iterator it(object.begin());
                it != object.end();
                ++it) {
                if (it != object.begin())
                    os << ",\n";
                os << prefix << quote(it->first) << " : ";
                boost::apply_visitor(*this, it->second);
            }
            --depth;
            prefix.clear();
            prefix.append(depth * 4, ' ');
            os << '\n' << prefix << "}";
        }
    }
    void operator()(const Array &array)
    {
        if (array.empty()) {
            os << "[ ]";
        } else {
            ++depth;
            std::string prefix(depth * 4, ' ');
            os << "[\n";
            for (Array::const_iterator it(array.begin());
                it != array.end();
                ++it) {
                if (it != array.begin())
                    os << ",\n";
                os << prefix;
                boost::apply_visitor(*this, *it);
            }
            --depth;
            prefix.clear();
            prefix.append(depth * 4, ' ');
            os << '\n' << prefix << "]";
        }
    }

    std::ostream &os;
    int depth;
};
}

std::ostream &operator <<(std::ostream &os, const Value &json)
{
    JSONVisitor visitor(os);
    boost::apply_visitor(visitor, json);
    return os;
}

}}

// Copyright (c) 2009 - Mozy, Inc.

#include "mordor/pch.h"

#include "mordor/json.h"

#include "mordor/assert.h"

namespace Mordor {
namespace JSON {

std::string unquote(const std::string &string)
{
    MORDOR_ASSERT(string.size() >= 2);
    MORDOR_ASSERT(string[0] == '"');
    MORDOR_ASSERT(string[string.size() - 1] == '"');
    std::string result = string.substr(1, string.size() - 2);

    const char *c = string.c_str() + 1;
    const char *end = c + string.size() - 2;
    bool differed = false;
    int utf16;
    while (c < end)
    {
        if (*c == '\\') {
            MORDOR_ASSERT(c + 1 < end);
            if (!differed) {
                result.resize(c - (string.c_str() + 1));
                differed = true;
            }
            ++c;
            switch (*c) {
                case '"':
                case '\\':
                case '/':
                    result.append(1, *c);
                    break;
                case 'b':
                    result.append(1, '\b');
                    break;
                case 'f':
                    result.append(1, '\f');
                    break;
                case 'n':
                    result.append(1, '\n');
                    break;
                case 'r':
                    result.append(1, '\r');
                    break;
                case 't':
                    result.append(1, '\t');
                    break;
                case 'u':
                    MORDOR_ASSERT(c + 4 < end);
                    utf16 = 0;
                    ++c;
                    for (int i = 0; i < 4; ++i) {
                        utf16 *= 16;
                        if (*c >= '0' && *c <= '9')
                            utf16 += *c - '0';
                        else if (*c >= 'a' && *c <= 'f')
                            utf16 += *c - 'a' + 10;
                        else if (*c >= 'A' && *c <= 'F')
                            utf16 += *c - 'A' + 10;
                        else
                            MORDOR_NOTREACHED();
                        ++c;
                    }
                    // Utf16->Utf8 conversion not supported yet
                    MORDOR_ASSERT(utf16 < 255);
                    result.append(1, (char)utf16);
                    break;
                default:
                    MORDOR_NOTREACHED();
            }
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

%%{
    machine json_parser;

    action mark { mark = fpc;}
    action done { fbreak; }
    prepush {
        prepush();
    }
    postpop {
        postpop();
    }

    ws = ' ' | '\t' | '\r' | '\n';

    unescaped = (any - ('"' | '\\') - cntrl);
    char = unescaped | ('\\' ('"' | '\\' | 'b' | 'f' | 'n' | 'r' | 't' | ('u' [0-9A-Za-z]{4})));
    string = '"' char* '"';

    action begin_number
    {
        m_nonIntegral = false;
    }
    action set_non_integral
    {
        m_nonIntegral = true;
    }
    action parse_number
    {
        if (m_nonIntegral) {
            *m_stack.top() = strtod(mark, NULL);
        } else {
            *m_stack.top() = strtoll(mark, NULL, 10);
        }
    }

    int = (digit | ([1-9] digit*));
    frac = '.' >set_non_integral digit+;
    exp = 'e'i >set_non_integral ('+' | '-')? digit+;
    number = ('-'? int frac? exp?) >mark >begin_number %parse_number;

    action parse_string
    {
        *m_stack.top() = unquote(std::string(mark, fpc - mark));
    }
    action call_parse_object
    {
        *m_stack.top() = Object();
        fcall *json_parser_en_parse_object;
    }
    action call_parse_array
    {
        *m_stack.top() = Array();
        fcall *json_parser_en_parse_array;
    }
    action parse_true
    {
        *m_stack.top() = true;
    }
    action parse_false
    {
        *m_stack.top() = false;
    }
    action parse_null
    {
        *m_stack.top() = boost::blank();
    }
    value = string >mark %parse_string | number | '{' @call_parse_object |
        '[' @call_parse_array | 'true' @parse_true | 'false' @parse_false |
        'null' @parse_null;

    object = '{' ws* string ws* ':' ws* value ws* (',' ws* string ws* ':' ws* value ws*)* '}';
    array = '[' ws* value ws* (',' ws* value ws*)* ']';

    action new_key
    {
        m_stack.push(&boost::get<Object>(*m_stack.top()).insert(std::make_pair(unquote(std::string(mark, fpc - mark)), Value()))->second);
    }
    action new_element
    {
        boost::get<Array>(*m_stack.top()).push_back(Value());
        m_stack.push(&boost::get<Array>(*m_stack.top()).back());
    }
    action pop_stack
    {
        m_stack.pop();
    }
    action ret {
        fret;
    }

    parse_object := parse_object_lbl: ws* string >mark %new_key ws* ':' ws* value %pop_stack ws* (',' ws* string >mark %new_key ws* ':' ws* value %pop_stack ws*)* '}' @ret;
    parse_array := parse_array_lbl: ws* value >new_element %pop_stack ws* (',' ws* value >new_element %pop_stack ws*)* ']' @ret;

    main := ws* '{' @call_parse_object | '[' @call_parse_array ws*;
    write data;
}%%

void
Parser::init()
{
    while (m_stack.size() > 1)
        m_stack.pop();
    *m_stack.top() = boost::blank();
    RagelParserWithStack::init();
    %% write init;
}

void
Parser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

bool
Parser::final() const
{
    return cs >= json_parser_first_final;
}

bool
Parser::error() const
{
    return cs == json_parser_error;
}

std::string
quote(const std::string &str)
{
    std::string result;
    result.reserve(str.length() + 2);
    result.append(1, '"');

    static const char *escaped =
        "\0\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
        "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
        "\\\"";

    size_t lastEscape = 0;
    size_t nextEscape = str.find_first_of(escaped, 0, 34);
    std::ostringstream os;
    os.fill('0');
    os << std::hex << std::setw(4);
    while (nextEscape != std::string::npos) {
        result.append(str.substr(lastEscape, nextEscape - lastEscape));
        result.append(1, '\\');
        switch (str[nextEscape]) {
            case '"':
            case '\\':
                result.append(1, str[nextEscape]);
                break;
            case '\b':
                result.append(1, 'b');
                break;
            case '\f':
                result.append(1, 'f');
                break;
            case '\n':
                result.append(1, 'n');
                break;
            case '\r':
                result.append(1, 'r');
                break;
            case '\t':
                result.append(1, 't');
                break;
            default:
                result.append(1, 'u');
                os.str();
                os << (int)str[nextEscape];
                result.append(os.str());
                break;
        }
        lastEscape = nextEscape + 1;
        nextEscape = str.find_first_of(escaped, lastEscape, 34);
    }
    result.append(str.substr(lastEscape));
    result.append(1, '"');
    return result;
}

namespace {
class JSONVisitor : public boost::static_visitor<>
{
public:
    JSONVisitor(std::ostream &os)
    : os(os),
      depth(0)
    {}

    void operator()(const boost::blank &b)
    {
        os << "null";
    }
    void operator()(const bool &b)
    {
        os << (b ? "true" : "false");
    }
    template <class T>
    void operator()(const T &t)
    {
        os << t;
    }
    void operator()(const std::string &str)
    {
        os << quote(str);
    }
    void operator()(const Object &object)
    {
        if (object.empty()) {
            os << "{ }";
        } else {
            ++depth;
            std::string prefix(depth * 4, ' ');
            os << "{\n";
            for (Object::const_iterator it(object.begin());
                it != object.end();
                ++it) {
                if (it != object.begin())
                    os << ",\n";
                os << prefix << quote(it->first) << " : ";
                boost::apply_visitor(*this, it->second);
            }
            --depth;
            prefix.clear();
            prefix.append(depth * 4, ' ');
            os << '\n' << prefix << "}";
        }
    }
    void operator()(const Array &array)
    {
        if (array.empty()) {
            os << "[ ]";
        } else {
            ++depth;
            std::string prefix(depth * 4, ' ');
            os << "[\n";
            for (Array::const_iterator it(array.begin());
                it != array.end();
                ++it) {
                if (it != array.begin())
                    os << ",\n";
                os << prefix;
                boost::apply_visitor(*this, *it);
            }
            --depth;
            prefix.clear();
            prefix.append(depth * 4, ' ');
            os << '\n' << prefix << "]";
        }
    }

    std::ostream &os;
    int depth;
};
}

std::ostream &operator <<(std::ostream &os, const Value &json)
{
    JSONVisitor visitor(os);
    boost::apply_visitor(visitor, json);
    return os;
}

}}