<commit_before>#include "test_compiler.hpp"

#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <sys/wait.h>
#include <unistd.h>

#include <cstdlib>
#include <regex>
#include <sstream>

#include <mettle/driver/scoped_pipe.hpp>
#include <mettle/driver/scoped_signal.hpp>
#include <mettle/driver/test_monitor.hpp>
#include <mettle/output.hpp>

#include "paths.hpp"

namespace caliber {

namespace {
  pid_t test_pgid = 0;
  struct sigaction old_sigint, old_sigquit;

  void sig_handler(int signum) {
    assert(test_pgid != 0);
    kill(-test_pgid, signum);

    // Restore the previous signal action and re-raise the signal.
    struct sigaction *old_act = signum == SIGINT ? &old_sigint : &old_sigquit;
    sigaction(signum, old_act, nullptr);
    raise(signum);
  }

  void sig_chld(int) {}

  inline std::string err_string(int errnum) {
    char buf[256];
#ifdef _GNU_SOURCE
    return strerror_r(errnum, buf, sizeof(buf));
#else
    if(strerror_r(errnum, buf, sizeof(buf)) < 0)
      return "";
    return buf;
#endif
  }

  inline mettle::test_result parent_failed() {
    return { false, err_string(errno) };
  }

  [[noreturn]] inline void child_failed() {
    _exit(128);
  }

  std::unique_ptr<char *[]>
  make_argv(const std::vector<std::string> &argv) {
    auto real_argv = std::make_unique<char *[]>(argv.size() + 1);
    for(size_t i = 0; i != argv.size(); i++)
      real_argv[i] = const_cast<char*>(argv[i].c_str());
    return real_argv;
  }
}

mettle::test_result
test_compiler::operator ()(
  const std::string &file, const compiler_options &args,
  const raw_options &raw_args, bool expect_fail,
  mettle::log::test_output &output
) const {
  mettle::scoped_pipe stdout_pipe, stderr_pipe;
  if(stdout_pipe.open() < 0 ||
     stderr_pipe.open() < 0)
    return parent_failed();

  std::string dir = parent_path(file);
  std::vector<std::string> final_args = {compiler_.path.c_str()};
  for(auto &&tok : translate_args(file, args, dir))
    final_args.push_back(std::move(tok));
  for(const auto &arg : raw_args) {
    if(tool_match(compiler_, arg.tool))
      final_args.push_back(arg.value);
  }

  fflush(nullptr);

  mettle::scoped_sigprocmask mask;
  if(mask.push(SIG_BLOCK, SIGCHLD) < 0 ||
     mask.push(SIG_BLOCK, {SIGINT, SIGQUIT}) < 0)
    return parent_failed();

  pid_t pid;
  if((pid = fork()) < 0)
    return parent_failed();

  if(pid == 0) {
    if(mask.clear() < 0)
      child_failed();

    // Make a new process group so we can kill the test and all its children
    // as a group.
    setpgid(0, 0);

    if(timeout_)
      mettle::fork_monitor(*timeout_);

    if(stdout_pipe.close_read() < 0 ||
       stderr_pipe.close_read() < 0)
      child_failed();

    if(stdout_pipe.move_write(STDOUT_FILENO) < 0 ||
       stderr_pipe.move_write(STDERR_FILENO) < 0)
      child_failed();

    execvp(compiler_.path.c_str(), make_argv(final_args).get());
    child_failed();
  }
  else {
    mettle::scoped_signal sigint, sigquit, sigchld;
    test_pgid = pid;

    if(sigaction(SIGINT, nullptr, &old_sigint) < 0 ||
       sigaction(SIGQUIT, nullptr, &old_sigquit) < 0)
      return parent_failed();

    if(sigint.open(SIGINT, sig_handler) < 0 ||
       sigquit.open(SIGQUIT, sig_handler) < 0 ||
       sigchld.open(SIGCHLD, sig_chld) < 0)
      return parent_failed();

    if(mask.pop() < 0)
      return parent_failed();

    if(stdout_pipe.close_write() < 0 ||
       stderr_pipe.close_write() < 0)
      return parent_failed();

    std::vector<mettle::readfd> dests = {
      {stdout_pipe.read_fd, &output.stdout},
      {stderr_pipe.read_fd, &output.stderr}
    };

    // Read from the piped stdout, stderr, and log. If we're interrupted
    // (probably by SIGCHLD), do one last non-blocking read to get any data we
    // might have missed.
    sigset_t empty;
    sigemptyset(&empty);
    if(mettle::read_into(dests, nullptr, &empty) < 0) {
      if(errno != EINTR)
        return parent_failed();
      timespec timeout = {0, 0};
      if(mettle::read_into(dests, &timeout, nullptr) < 0)
        return parent_failed();
    }

    int status;
    if(waitpid(pid, &status, 0) < 0)
      return parent_failed();

    // Make sure everything in the test's process group is dead. Don't worry
    // about reaping.
    kill(-pid, SIGKILL);
    test_pgid = 0;

    if(WIFEXITED(status)) {
      int exit_code = WEXITSTATUS(status);
      if(exit_code == mettle::err_timeout) {
        std::ostringstream ss;
        ss << "Timed out after " << timeout_->count() << " ms";
        return { false, ss.str() };
      }
      else if(bool(exit_code) == expect_fail) {
        return { true, "" };
      }
      else if(exit_code) {
        return { false, "Compilation failed" };
      }
      else {
        return { false, "Compilation successful" };
      }
    }
    else if(WIFSIGNALED(status)) {
      return { false, strsignal(WTERMSIG(status)) };
    }
    else { // WIFSTOPPED
      kill(pid, SIGKILL);
      return { false, strsignal(WSTOPSIG(status)) };
    }
  }
}

} // namespace caliber
<commit_msg>Rename stdout/stderr to match mettle's changes<commit_after>#include "test_compiler.hpp"

#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <sys/wait.h>
#include <unistd.h>

#include <cstdlib>
#include <regex>
#include <sstream>

#include <mettle/driver/scoped_pipe.hpp>
#include <mettle/driver/scoped_signal.hpp>
#include <mettle/driver/test_monitor.hpp>
#include <mettle/output.hpp>

#include "paths.hpp"

namespace caliber {

namespace {
  pid_t test_pgid = 0;
  struct sigaction old_sigint, old_sigquit;

  void sig_handler(int signum) {
    assert(test_pgid != 0);
    kill(-test_pgid, signum);

    // Restore the previous signal action and re-raise the signal.
    struct sigaction *old_act = signum == SIGINT ? &old_sigint : &old_sigquit;
    sigaction(signum, old_act, nullptr);
    raise(signum);
  }

  void sig_chld(int) {}

  inline std::string err_string(int errnum) {
    char buf[256];
#ifdef _GNU_SOURCE
    return strerror_r(errnum, buf, sizeof(buf));
#else
    if(strerror_r(errnum, buf, sizeof(buf)) < 0)
      return "";
    return buf;
#endif
  }

  inline mettle::test_result parent_failed() {
    return { false, err_string(errno) };
  }

  [[noreturn]] inline void child_failed() {
    _exit(128);
  }

  std::unique_ptr<char *[]>
  make_argv(const std::vector<std::string> &argv) {
    auto real_argv = std::make_unique<char *[]>(argv.size() + 1);
    for(size_t i = 0; i != argv.size(); i++)
      real_argv[i] = const_cast<char*>(argv[i].c_str());
    return real_argv;
  }
}

mettle::test_result
test_compiler::operator ()(
  const std::string &file, const compiler_options &args,
  const raw_options &raw_args, bool expect_fail,
  mettle::log::test_output &output
) const {
  mettle::scoped_pipe stdout_pipe, stderr_pipe;
  if(stdout_pipe.open() < 0 ||
     stderr_pipe.open() < 0)
    return parent_failed();

  std::string dir = parent_path(file);
  std::vector<std::string> final_args = {compiler_.path.c_str()};
  for(auto &&tok : translate_args(file, args, dir))
    final_args.push_back(std::move(tok));
  for(const auto &arg : raw_args) {
    if(tool_match(compiler_, arg.tool))
      final_args.push_back(arg.value);
  }

  fflush(nullptr);

  mettle::scoped_sigprocmask mask;
  if(mask.push(SIG_BLOCK, SIGCHLD) < 0 ||
     mask.push(SIG_BLOCK, {SIGINT, SIGQUIT}) < 0)
    return parent_failed();

  pid_t pid;
  if((pid = fork()) < 0)
    return parent_failed();

  if(pid == 0) {
    if(mask.clear() < 0)
      child_failed();

    // Make a new process group so we can kill the test and all its children
    // as a group.
    setpgid(0, 0);

    if(timeout_)
      mettle::fork_monitor(*timeout_);

    if(stdout_pipe.close_read() < 0 ||
       stderr_pipe.close_read() < 0)
      child_failed();

    if(stdout_pipe.move_write(STDOUT_FILENO) < 0 ||
       stderr_pipe.move_write(STDERR_FILENO) < 0)
      child_failed();

    execvp(compiler_.path.c_str(), make_argv(final_args).get());
    child_failed();
  }
  else {
    mettle::scoped_signal sigint, sigquit, sigchld;
    test_pgid = pid;

    if(sigaction(SIGINT, nullptr, &old_sigint) < 0 ||
       sigaction(SIGQUIT, nullptr, &old_sigquit) < 0)
      return parent_failed();

    if(sigint.open(SIGINT, sig_handler) < 0 ||
       sigquit.open(SIGQUIT, sig_handler) < 0 ||
       sigchld.open(SIGCHLD, sig_chld) < 0)
      return parent_failed();

    if(mask.pop() < 0)
      return parent_failed();

    if(stdout_pipe.close_write() < 0 ||
       stderr_pipe.close_write() < 0)
      return parent_failed();

    std::vector<mettle::readfd> dests = {
      {stdout_pipe.read_fd, &output.stdout_log},
      {stderr_pipe.read_fd, &output.stderr_log}
    };

    // Read from the piped stdout, stderr, and log. If we're interrupted
    // (probably by SIGCHLD), do one last non-blocking read to get any data we
    // might have missed.
    sigset_t empty;
    sigemptyset(&empty);
    if(mettle::read_into(dests, nullptr, &empty) < 0) {
      if(errno != EINTR)
        return parent_failed();
      timespec timeout = {0, 0};
      if(mettle::read_into(dests, &timeout, nullptr) < 0)
        return parent_failed();
    }

    int status;
    if(waitpid(pid, &status, 0) < 0)
      return parent_failed();

    // Make sure everything in the test's process group is dead. Don't worry
    // about reaping.
    kill(-pid, SIGKILL);
    test_pgid = 0;

    if(WIFEXITED(status)) {
      int exit_code = WEXITSTATUS(status);
      if(exit_code == mettle::err_timeout) {
        std::ostringstream ss;
        ss << "Timed out after " << timeout_->count() << " ms";
        return { false, ss.str() };
      }
      else if(bool(exit_code) == expect_fail) {
        return { true, "" };
      }
      else if(exit_code) {
        return { false, "Compilation failed" };
      }
      else {
        return { false, "Compilation successful" };
      }
    }
    else if(WIFSIGNALED(status)) {
      return { false, strsignal(WTERMSIG(status)) };
    }
    else { // WIFSTOPPED
      kill(pid, SIGKILL);
      return { false, strsignal(WSTOPSIG(status)) };
    }
  }
}

} // namespace caliber
<|endoftext|>


<commit_before>#include "crt.h"
#include "scanner.h"
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include "scanscalar.h"
#include <sstream>

namespace YAML
{
	///////////////////////////////////////////////////////////////////////
	// Specialization for scanning specific tokens

	// Directive
	// . Note: no semantic checking is done here (that's for the parser to do)
	void Scanner::ScanDirective()
	{
		std::string name;
		std::vector <std::string> params;

		// pop indents and simple keys
		PopAllIndents();
		PopAllSimpleKeys();

		m_simpleKeyAllowed = false;

		// store pos and eat indicator
		Mark mark = INPUT.mark();
		INPUT.eat(1);

		// read name
		while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
			name += INPUT.get();

		// read parameters
		while(1) {
			// first get rid of whitespace
			while(Exp::Blank.Matches(INPUT))
				INPUT.eat(1);

			// break on newline or comment
			if(!INPUT || Exp::Break.Matches(INPUT) || Exp::Comment.Matches(INPUT))
				break;

			// now read parameter
			std::string param;
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
				param += INPUT.get();

			params.push_back(param);
		}

		Token token(Token::DIRECTIVE, mark);
		token.value = name;
		token.params = params;
		m_tokens.push(token);
	}

	// DocStart
	void Scanner::ScanDocStart()
	{
		PopAllIndents();
		PopAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(3);
		m_tokens.push(Token(Token::DOC_START, mark));
	}

	// DocEnd
	void Scanner::ScanDocEnd()
	{
		PopAllIndents();
		PopAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(3);
		m_tokens.push(Token(Token::DOC_END, mark));
	}

	// FlowStart
	void Scanner::ScanFlowStart()
	{
		// flows can be simple keys
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		char ch = INPUT.get();
		FLOW_MARKER flowType = (ch == Keys::FlowSeqStart ? FLOW_SEQ : FLOW_MAP);
		m_flows.push(flowType);
		Token::TYPE type = (flowType == FLOW_SEQ ? Token::FLOW_SEQ_START : Token::FLOW_MAP_START);
		m_tokens.push(Token(type, mark));
	}

	// FlowEnd
	void Scanner::ScanFlowEnd()
	{
		if(InBlockContext())
			throw ParserException(INPUT.mark(), ErrorMsg::FLOW_END);

		// we might have a solo entry in the flow context
		if(VerifySimpleKey())
			m_tokens.push(Token(Token::VALUE, INPUT.mark()));

		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		char ch = INPUT.get();

		// check that it matches the start
		FLOW_MARKER flowType = (ch == Keys::FlowSeqEnd ? FLOW_SEQ : FLOW_MAP);
		if(m_flows.top() != flowType)
			throw ParserException(mark, ErrorMsg::FLOW_END);
		m_flows.pop();

		Token::TYPE type = (flowType ? Token::FLOW_SEQ_END : Token::FLOW_MAP_END);
		m_tokens.push(Token(type, mark));
	}

	// FlowEntry
	void Scanner::ScanFlowEntry()
	{
		 // we might have a solo entry in the flow context
		if(VerifySimpleKey())
			m_tokens.push(Token(Token::VALUE, INPUT.mark()));

		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::FLOW_ENTRY, mark));
	}

	// BlockEntry
	void Scanner::ScanBlockEntry()
	{
		// we better be in the block context!
		if(InFlowContext())
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);

		// can we put it here?
		if(!m_simpleKeyAllowed)
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);

		PushIndentTo(INPUT.column(), IndentMarker::SEQ);
		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::BLOCK_ENTRY, mark));
	}

	// Key
	void Scanner::ScanKey()
	{
		// handle keys diffently in the block context (and manage indents)
		if(InBlockContext()) {
			if(!m_simpleKeyAllowed)
				throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY);

			PushIndentTo(INPUT.column(), IndentMarker::MAP);
		}

		// can only put a simple key here if we're in block context
		m_simpleKeyAllowed = InBlockContext();

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::KEY, mark));
	}

	// Value
	void Scanner::ScanValue()
	{
		// and check that simple key
		bool isSimpleKey = VerifySimpleKey();

		if(isSimpleKey) {
			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
			m_simpleKeyAllowed = false;
		} else {
			// handle values diffently in the block context (and manage indents)
			if(InBlockContext()) {
				if(!m_simpleKeyAllowed)
					throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE);

				PushIndentTo(INPUT.column(), IndentMarker::MAP);
			}

			// can only put a simple key here if we're in block context
			m_simpleKeyAllowed = InBlockContext();
		}

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::VALUE, mark));
	}

	// AnchorOrAlias
	void Scanner::ScanAnchorOrAlias()
	{
		bool alias;
		std::string name;

		// insert a potential simple key
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
		Mark mark = INPUT.mark();
		char indicator = INPUT.get();
		alias = (indicator == Keys::Alias);

		// now eat the content
		while(Exp::AlphaNumeric.Matches(INPUT))
			name += INPUT.get();

		// we need to have read SOMETHING!
		if(name.empty())
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::ALIAS_NOT_FOUND : ErrorMsg::ANCHOR_NOT_FOUND);

		// and needs to end correctly
		if(INPUT && !Exp::AnchorEnd.Matches(INPUT))
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::CHAR_IN_ALIAS : ErrorMsg::CHAR_IN_ANCHOR);

		// and we're done
		Token token(alias ? Token::ALIAS : Token::ANCHOR, mark);
		token.value = name;
		m_tokens.push(token);
	}

	// Tag
	void Scanner::ScanTag()
	{
		std::string handle, suffix;

		// insert a potential simple key
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
		Mark mark = INPUT.mark();
		handle += INPUT.get();

		// read the handle
		while(INPUT && INPUT.peek() != Keys::Tag && !Exp::BlankOrBreak.Matches(INPUT))
			handle += INPUT.get();

		// is there a suffix?
		if(INPUT.peek() == Keys::Tag) {
			// eat the indicator
			handle += INPUT.get();

			// then read it
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
				suffix += INPUT.get();
		} else {
			// this is a bit weird: we keep just the '!' as the handle and move the rest to the suffix
			suffix = handle.substr(1);
			handle = "!";
		}

		Token token(Token::TAG, mark);
		token.value = handle;
		token.params.push_back(suffix);
		m_tokens.push(token);
	}

	// PlainScalar
	void Scanner::ScanPlainScalar()
	{
		std::string scalar;

		// set up the scanning parameters
		ScanScalarParams params;
		params.end = (InFlowContext() ? Exp::EndScalarInFlow : Exp::EndScalar) || (Exp::BlankOrBreak + Exp::Comment);
		params.eatEnd = false;
		params.indent = (InFlowContext() ? 0 : GetTopIndent() + 1);
		params.fold = FOLD_BLOCK;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = true;
		params.chomp = STRIP;
		params.onDocIndicator = BREAK;
		params.onTabInIndentation = THROW;

		// insert a potential simple key
		InsertPotentialSimpleKey();

		Mark mark = INPUT.mark();
		scalar = ScanScalar(INPUT, params);

		// can have a simple key only if we ended the scalar by starting a new line
		m_simpleKeyAllowed = params.leadingSpaces;

		// finally, check and see if we ended on an illegal character
		//if(Exp::IllegalCharInScalar.Matches(INPUT))
		//	throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}

	// QuotedScalar
	void Scanner::ScanQuotedScalar()
	{
		std::string scalar;

		// peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
		char quote = INPUT.peek();
		bool single = (quote == '\'');

		// setup the scanning parameters
		ScanScalarParams params;
		params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
		params.eatEnd = true;
		params.escape = (single ? '\'' : '\\');
		params.indent = 0;
		params.fold = FOLD_FLOW;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = false;
		params.chomp = CLIP;
		params.onDocIndicator = THROW;

		// insert a potential simple key
		InsertPotentialSimpleKey();

		Mark mark = INPUT.mark();

		// now eat that opening quote
		INPUT.get();

		// and scan
		scalar = ScanScalar(INPUT, params);
		m_simpleKeyAllowed = false;

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}

	// BlockScalarToken
	// . These need a little extra processing beforehand.
	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
	//   and then we need to figure out what level of indentation we'll be using.
	void Scanner::ScanBlockScalar()
	{
		std::string scalar;

		ScanScalarParams params;
		params.indent = 1;
		params.detectIndent = true;

		// eat block indicator ('|' or '>')
		Mark mark = INPUT.mark();
		char indicator = INPUT.get();
		params.fold = (indicator == Keys::FoldedScalar ? FOLD_BLOCK : DONT_FOLD);

		// eat chomping/indentation indicators
		params.chomp = CLIP;
		int n = Exp::Chomp.Match(INPUT);
		for(int i=0;i<n;i++) {
			char ch = INPUT.get();
			if(ch == '+')
				params.chomp = KEEP;
			else if(ch == '-')
				params.chomp = STRIP;
			else if(Exp::Digit.Matches(ch)) {
				if(ch == '0')
					throw ParserException(INPUT.mark(), ErrorMsg::ZERO_INDENT_IN_BLOCK);

				params.indent = ch - '0';
				params.detectIndent = false;
			}
		}

		// now eat whitespace
		while(Exp::Blank.Matches(INPUT))
			INPUT.eat(1);

		// and comments to the end of the line
		if(Exp::Comment.Matches(INPUT))
			while(INPUT && !Exp::Break.Matches(INPUT))
				INPUT.eat(1);

		// if it's not a line break, then we ran into a bad character inline
		if(INPUT && !Exp::Break.Matches(INPUT))
			throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK);

		// set the initial indentation
		if(GetTopIndent() >= 0)
			params.indent += GetTopIndent();

		params.eatLeadingWhitespace = false;
		params.trimTrailingSpaces = false;
		params.onTabInIndentation = THROW;

		scalar = ScanScalar(INPUT, params);

		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
		m_simpleKeyAllowed = true;

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}
}
<commit_msg>Fixed bug in plain scalar folding<commit_after>#include "crt.h"
#include "scanner.h"
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include "scanscalar.h"
#include <sstream>

namespace YAML
{
	///////////////////////////////////////////////////////////////////////
	// Specialization for scanning specific tokens

	// Directive
	// . Note: no semantic checking is done here (that's for the parser to do)
	void Scanner::ScanDirective()
	{
		std::string name;
		std::vector <std::string> params;

		// pop indents and simple keys
		PopAllIndents();
		PopAllSimpleKeys();

		m_simpleKeyAllowed = false;

		// store pos and eat indicator
		Mark mark = INPUT.mark();
		INPUT.eat(1);

		// read name
		while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
			name += INPUT.get();

		// read parameters
		while(1) {
			// first get rid of whitespace
			while(Exp::Blank.Matches(INPUT))
				INPUT.eat(1);

			// break on newline or comment
			if(!INPUT || Exp::Break.Matches(INPUT) || Exp::Comment.Matches(INPUT))
				break;

			// now read parameter
			std::string param;
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
				param += INPUT.get();

			params.push_back(param);
		}

		Token token(Token::DIRECTIVE, mark);
		token.value = name;
		token.params = params;
		m_tokens.push(token);
	}

	// DocStart
	void Scanner::ScanDocStart()
	{
		PopAllIndents();
		PopAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(3);
		m_tokens.push(Token(Token::DOC_START, mark));
	}

	// DocEnd
	void Scanner::ScanDocEnd()
	{
		PopAllIndents();
		PopAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(3);
		m_tokens.push(Token(Token::DOC_END, mark));
	}

	// FlowStart
	void Scanner::ScanFlowStart()
	{
		// flows can be simple keys
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		char ch = INPUT.get();
		FLOW_MARKER flowType = (ch == Keys::FlowSeqStart ? FLOW_SEQ : FLOW_MAP);
		m_flows.push(flowType);
		Token::TYPE type = (flowType == FLOW_SEQ ? Token::FLOW_SEQ_START : Token::FLOW_MAP_START);
		m_tokens.push(Token(type, mark));
	}

	// FlowEnd
	void Scanner::ScanFlowEnd()
	{
		if(InBlockContext())
			throw ParserException(INPUT.mark(), ErrorMsg::FLOW_END);

		// we might have a solo entry in the flow context
		if(VerifySimpleKey())
			m_tokens.push(Token(Token::VALUE, INPUT.mark()));

		m_simpleKeyAllowed = false;

		// eat
		Mark mark = INPUT.mark();
		char ch = INPUT.get();

		// check that it matches the start
		FLOW_MARKER flowType = (ch == Keys::FlowSeqEnd ? FLOW_SEQ : FLOW_MAP);
		if(m_flows.top() != flowType)
			throw ParserException(mark, ErrorMsg::FLOW_END);
		m_flows.pop();

		Token::TYPE type = (flowType ? Token::FLOW_SEQ_END : Token::FLOW_MAP_END);
		m_tokens.push(Token(type, mark));
	}

	// FlowEntry
	void Scanner::ScanFlowEntry()
	{
		 // we might have a solo entry in the flow context
		if(VerifySimpleKey())
			m_tokens.push(Token(Token::VALUE, INPUT.mark()));

		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::FLOW_ENTRY, mark));
	}

	// BlockEntry
	void Scanner::ScanBlockEntry()
	{
		// we better be in the block context!
		if(InFlowContext())
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);

		// can we put it here?
		if(!m_simpleKeyAllowed)
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);

		PushIndentTo(INPUT.column(), IndentMarker::SEQ);
		m_simpleKeyAllowed = true;

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::BLOCK_ENTRY, mark));
	}

	// Key
	void Scanner::ScanKey()
	{
		// handle keys diffently in the block context (and manage indents)
		if(InBlockContext()) {
			if(!m_simpleKeyAllowed)
				throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY);

			PushIndentTo(INPUT.column(), IndentMarker::MAP);
		}

		// can only put a simple key here if we're in block context
		m_simpleKeyAllowed = InBlockContext();

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::KEY, mark));
	}

	// Value
	void Scanner::ScanValue()
	{
		// and check that simple key
		bool isSimpleKey = VerifySimpleKey();

		if(isSimpleKey) {
			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
			m_simpleKeyAllowed = false;
		} else {
			// handle values diffently in the block context (and manage indents)
			if(InBlockContext()) {
				if(!m_simpleKeyAllowed)
					throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE);

				PushIndentTo(INPUT.column(), IndentMarker::MAP);
			}

			// can only put a simple key here if we're in block context
			m_simpleKeyAllowed = InBlockContext();
		}

		// eat
		Mark mark = INPUT.mark();
		INPUT.eat(1);
		m_tokens.push(Token(Token::VALUE, mark));
	}

	// AnchorOrAlias
	void Scanner::ScanAnchorOrAlias()
	{
		bool alias;
		std::string name;

		// insert a potential simple key
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
		Mark mark = INPUT.mark();
		char indicator = INPUT.get();
		alias = (indicator == Keys::Alias);

		// now eat the content
		while(Exp::AlphaNumeric.Matches(INPUT))
			name += INPUT.get();

		// we need to have read SOMETHING!
		if(name.empty())
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::ALIAS_NOT_FOUND : ErrorMsg::ANCHOR_NOT_FOUND);

		// and needs to end correctly
		if(INPUT && !Exp::AnchorEnd.Matches(INPUT))
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::CHAR_IN_ALIAS : ErrorMsg::CHAR_IN_ANCHOR);

		// and we're done
		Token token(alias ? Token::ALIAS : Token::ANCHOR, mark);
		token.value = name;
		m_tokens.push(token);
	}

	// Tag
	void Scanner::ScanTag()
	{
		std::string handle, suffix;

		// insert a potential simple key
		InsertPotentialSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
		Mark mark = INPUT.mark();
		handle += INPUT.get();

		// read the handle
		while(INPUT && INPUT.peek() != Keys::Tag && !Exp::BlankOrBreak.Matches(INPUT))
			handle += INPUT.get();

		// is there a suffix?
		if(INPUT.peek() == Keys::Tag) {
			// eat the indicator
			handle += INPUT.get();

			// then read it
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
				suffix += INPUT.get();
		} else {
			// this is a bit weird: we keep just the '!' as the handle and move the rest to the suffix
			suffix = handle.substr(1);
			handle = "!";
		}

		Token token(Token::TAG, mark);
		token.value = handle;
		token.params.push_back(suffix);
		m_tokens.push(token);
	}

	// PlainScalar
	void Scanner::ScanPlainScalar()
	{
		std::string scalar;

		// set up the scanning parameters
		ScanScalarParams params;
		params.end = (InFlowContext() ? Exp::EndScalarInFlow : Exp::EndScalar) || (Exp::BlankOrBreak + Exp::Comment);
		params.eatEnd = false;
		params.indent = (InFlowContext() ? 0 : GetTopIndent() + 1);
		params.fold = FOLD_FLOW;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = true;
		params.chomp = STRIP;
		params.onDocIndicator = BREAK;
		params.onTabInIndentation = THROW;

		// insert a potential simple key
		InsertPotentialSimpleKey();

		Mark mark = INPUT.mark();
		scalar = ScanScalar(INPUT, params);

		// can have a simple key only if we ended the scalar by starting a new line
		m_simpleKeyAllowed = params.leadingSpaces;

		// finally, check and see if we ended on an illegal character
		//if(Exp::IllegalCharInScalar.Matches(INPUT))
		//	throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}

	// QuotedScalar
	void Scanner::ScanQuotedScalar()
	{
		std::string scalar;

		// peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
		char quote = INPUT.peek();
		bool single = (quote == '\'');

		// setup the scanning parameters
		ScanScalarParams params;
		params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
		params.eatEnd = true;
		params.escape = (single ? '\'' : '\\');
		params.indent = 0;
		params.fold = FOLD_FLOW;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = false;
		params.chomp = CLIP;
		params.onDocIndicator = THROW;

		// insert a potential simple key
		InsertPotentialSimpleKey();

		Mark mark = INPUT.mark();

		// now eat that opening quote
		INPUT.get();

		// and scan
		scalar = ScanScalar(INPUT, params);
		m_simpleKeyAllowed = false;

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}

	// BlockScalarToken
	// . These need a little extra processing beforehand.
	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
	//   and then we need to figure out what level of indentation we'll be using.
	void Scanner::ScanBlockScalar()
	{
		std::string scalar;

		ScanScalarParams params;
		params.indent = 1;
		params.detectIndent = true;

		// eat block indicator ('|' or '>')
		Mark mark = INPUT.mark();
		char indicator = INPUT.get();
		params.fold = (indicator == Keys::FoldedScalar ? FOLD_BLOCK : DONT_FOLD);

		// eat chomping/indentation indicators
		params.chomp = CLIP;
		int n = Exp::Chomp.Match(INPUT);
		for(int i=0;i<n;i++) {
			char ch = INPUT.get();
			if(ch == '+')
				params.chomp = KEEP;
			else if(ch == '-')
				params.chomp = STRIP;
			else if(Exp::Digit.Matches(ch)) {
				if(ch == '0')
					throw ParserException(INPUT.mark(), ErrorMsg::ZERO_INDENT_IN_BLOCK);

				params.indent = ch - '0';
				params.detectIndent = false;
			}
		}

		// now eat whitespace
		while(Exp::Blank.Matches(INPUT))
			INPUT.eat(1);

		// and comments to the end of the line
		if(Exp::Comment.Matches(INPUT))
			while(INPUT && !Exp::Break.Matches(INPUT))
				INPUT.eat(1);

		// if it's not a line break, then we ran into a bad character inline
		if(INPUT && !Exp::Break.Matches(INPUT))
			throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK);

		// set the initial indentation
		if(GetTopIndent() >= 0)
			params.indent += GetTopIndent();

		params.eatLeadingWhitespace = false;
		params.trimTrailingSpaces = false;
		params.onTabInIndentation = THROW;

		scalar = ScanScalar(INPUT, params);

		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
		m_simpleKeyAllowed = true;

		Token token(Token::SCALAR, mark);
		token.value = scalar;
		m_tokens.push(token);
	}
}
<|endoftext|>


<commit_before>#line 2 "togo/hash_map.hpp"
/**
@copyright MIT license; see @ref index or the accompanying LICENSE file.

@file hash_map.hpp
@brief HashMap interface.
@ingroup collections
@ingroup hash_map
*/

#pragma once

#include <togo/config.hpp>
#include <togo/types.hpp>
#include <togo/collection_types.hpp>
#include <togo/utility.hpp>
#include <togo/assert.hpp>
#include <togo/memory.hpp>
#include <togo/array.hpp>

#include <utility>

namespace togo {

/// Construct with allocator for storage.
template<class K, class T>
inline HashMap<K, T>::HashMap(Allocator& allocator)
	: _head(allocator)
	, _data(allocator)
{}

namespace hash_map {

/**
	@addtogroup hash_map
	@{
*/

/** @cond INTERNAL */
#define TOGO_HASH_MAP_MAX_LOAD 0.70f
/** @endcond */ // INTERNAL

/// Number of partitions.
template<class K, class T>
inline u32_fast num_partitions(HashMap<K, T> const& hm) { return array::size(hm._head); }

/// Number of values.
template<class K, class T>
inline u32_fast size(HashMap<K, T> const& hm) { return array::size(hm._data); }

/// Number of values reserved.
template<class K, class T>
inline u32_fast capacity(HashMap<K, T> const& hm) {
	return static_cast<u32_fast>(num_partitions(hm) * TOGO_HASH_MAP_MAX_LOAD);
}

/// Number of values that can be added before a resize occurs.
template<class K, class T>
inline u32_fast space(HashMap<K, T> const& hm) {
	return capacity(hm) - size(hm);
}

/// Returns true if there are any values.
template<class K, class T>
inline bool any(HashMap<K, T> const& hm) { return size(hm) != 0; }

/// Returns true if there are no values.
template<class K, class T>
inline bool empty(HashMap<K, T> const& hm) { return size(hm) == 0; }

/// Beginning iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T>* begin(HashMap<K, T>& hm) {
	return array::begin(hm._data);
}
/// Beginning iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T> const* begin(HashMap<K, T> const& hm) {
	return array::begin(hm._data);
}

/// Ending iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T>* end(HashMap<K, T>& hm) {
	return array::end(hm._data);
}
/// Ending iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T> const* end(HashMap<K, T> const& hm) {
	return array::end(hm._data);
}

/** @cond INTERNAL */
enum : unsigned {
	END = ~0u,
};

struct FindData {
	// Index into _head
	u32 head;
	// Index into _data
	u32 data;
	// Previous node with same key
	u32 data_prev;
};

namespace internal {

template<class K, class T>
FindData find(HashMap<K, T> const& hm, K const key) {
	FindData fd{END, END, END};
	if (array::empty(hm._head)) {
		return fd;
	}

	fd.head = key % num_partitions(hm);
	fd.data = hm._head[fd.head];
	while (fd.data != END) {
		auto const& node = hm._data[fd.data];
		if (node.key == key) {
			break;
		}
		fd.data_prev = fd.data;
		fd.data = node.next;
	}
	return fd;
}

template<class K, class T>
FindData find(
	HashMap<K, T> const& hm,
	HashMapNode<K, T> const* node
) {
	FindData fd{END, END, END};
	if (array::empty(hm._head)) {
		return fd;
	}

	fd.head = node->key % num_partitions(hm);
	fd.data = hm._head[fd.head];
	while (fd.data != END) {
		auto const& it_node = hm._data[fd.data];
		if (&it_node == node) {
			break;
		}
		fd.data_prev = fd.data;
		fd.data = it_node.next;
	}
	return fd;
}

template<class K, class T>
u32 make(HashMap<K, T>& hm, K const key, bool const keep) {
	TOGO_DEBUG_ASSERTE(array::any(hm._head));
	FindData const fd = find(hm, key);
	if (keep && fd.data != END) {
		return fd.data;
	}
	u32 const index = array::size(hm._data);
	HashMapNode<K, T> node;
	node.key = key;
	node.next = fd.data;
	array::push_back(hm._data, node);
	if (fd.data_prev == END) {
		hm._head[fd.head] = index;
	} else {
		hm._data[fd.data_prev].next = index;
	}
	return index;
}

template<class K, class T>
void remove(HashMap<K, T>& hm, FindData const& fd) {
	if (fd.data_prev == END) {
		// find() was immediate hit; fd is the first item
		// in the head. Move the head to the next item (if any).
		hm._head[fd.head] = hm._data[fd.data].next;
	} else {
		hm._data[fd.data_prev].next = hm._data[fd.data].next;
	}

	// Move last node to the removed position
	if (fd.data != array::size(hm._data) - 1) {
		auto const& last_node = array::back(hm._data);
		FindData const last = find(hm, &last_node);
		if (last.data_prev == END) {
			hm._head[last.head] = fd.data;
		} else {
			hm._data[last.data_prev].next = fd.data;
		}
		hm._data[fd.data] = last_node;
	}
	array::pop_back(hm._data);
}

template<class K, class T>
void resize(HashMap<K, T>& hm, u32_fast const new_size) {
	HashMap<K, T> new_hm{*hm._head._allocator};
	array::resize(new_hm._head, new_size);
	array::reserve(new_hm._data, min(new_size, array::capacity(hm._data)));
	for (unsigned i = 0; i < new_size; ++i) {
		new_hm._head[i] = END;
	}
	u32 index;
	for (auto const& node : hm._data) {
		index = make(hm, node.key, false);
		hm._data[index].value = node.value;
	}
	hm = std::move(new_hm);
}

template<class K, class T>
inline void grow(HashMap<K, T>& hm) {
	resize(hm, num_partitions(hm) * 2 + 16);
}

} // namespace internal
/** @endcond */ // INTERNAL

/// Reserve at least new_capacity.
template<class K, class T>
inline void reserve(HashMap<K, T>& hm, u32_fast const new_capacity) {
	if (new_capacity > capacity(hm)) {
		internal::resize(hm, (new_capacity / TOGO_HASH_MAP_MAX_LOAD) + 1);
	}
}

/// Remove all items.
template<class K, class T>
inline void clear(HashMap<K, T>& hm) {
	auto const size = num_partitions(hm);
	for (unsigned i = 0; i < size; ++i) {
		hm._head[i] = END;
	}
	array::clear(hm._data);
}

/// Set item.
///
/// If key does not exist, it will be inserted.
template<class K, class T>
inline void set(HashMap<K, T>& hm, K const key, T const& value) {
	if (!space(hm)) {
		internal::grow(hm);
	}
	auto const index = internal::make(hm, key, true);
	hm._data[index].value = value;
}

/// Get item.
///
/// If key does not exist, nullptr will be returned.
template<class K, class T>
inline T* get(HashMap<K, T>& hm, K const key) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index].value : nullptr;
}

/// Get item.
///
/// If key does not exist, nullptr will be returned.
template<class K, class T>
inline T const* get(HashMap<K, T> const& hm, K const key) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index].value : nullptr;
}

/// Check if there is a value with key.
template<class K, class T>
inline bool has(HashMap<K, T> const& hm, K const key) {
	return internal::find(hm, key).data != END;
}

/// Remove value.
template<class K, class T>
inline void remove(HashMap<K, T>& hm, K const key) {
	FindData const fd = internal::find(hm, key);
	if (fd.data != END) {
		internal::remove(hm, fd);
	}
}

/** @cond INTERNAL */
#undef TOGO_HASH_MAP_MAX_LOAD
/** @endcond */ // INTERNAL

/** @} */ // end of doc-group hash_map

} // namespace hash_map

/** @cond INTERNAL */

// ADL support

template<class K, class T>
inline HashMapNode<K, T>* begin(HashMap<K, T>& hm) {
	return hash_map::begin(hm);
}
template<class K, class T>
inline HashMapNode<K, T> const* begin(HashMap<K, T> const& hm) {
	return hash_map::begin(hm);
}

template<class K, class T>
inline HashMapNode<K, T>* end(HashMap<K, T>& hm) {
	return hash_map::end(hm);
}
template<class K, class T>
inline HashMapNode<K, T> const* end(HashMap<K, T> const& hm) {
	return hash_map::end(hm);
}

/** @endcond */ // INTERNAL

} // namespace togo
<commit_msg>hash_map: added node access interface; doc tidy.<commit_after>#line 2 "togo/hash_map.hpp"
/**
@copyright MIT license; see @ref index or the accompanying LICENSE file.

@file hash_map.hpp
@brief HashMap interface.
@ingroup collections
@ingroup hash_map
*/

#pragma once

#include <togo/config.hpp>
#include <togo/types.hpp>
#include <togo/collection_types.hpp>
#include <togo/utility.hpp>
#include <togo/assert.hpp>
#include <togo/memory.hpp>
#include <togo/array.hpp>

#include <utility>

namespace togo {

/// Construct with allocator for storage.
template<class K, class T>
inline HashMap<K, T>::HashMap(Allocator& allocator)
	: _head(allocator)
	, _data(allocator)
{}

namespace hash_map {

/**
	@addtogroup hash_map
	@{
*/

/** @cond INTERNAL */
#define TOGO_HASH_MAP_MAX_LOAD 0.70f
/** @endcond */ // INTERNAL

/// Number of partitions.
template<class K, class T>
inline u32_fast num_partitions(HashMap<K, T> const& hm) { return array::size(hm._head); }

/// Number of values.
template<class K, class T>
inline u32_fast size(HashMap<K, T> const& hm) { return array::size(hm._data); }

/// Number of values reserved.
template<class K, class T>
inline u32_fast capacity(HashMap<K, T> const& hm) {
	return static_cast<u32_fast>(num_partitions(hm) * TOGO_HASH_MAP_MAX_LOAD);
}

/// Number of values that can be added before a resize occurs.
template<class K, class T>
inline u32_fast space(HashMap<K, T> const& hm) {
	return capacity(hm) - size(hm);
}

/// Returns true if there are any values.
template<class K, class T>
inline bool any(HashMap<K, T> const& hm) { return size(hm) != 0; }

/// Returns true if there are no values.
template<class K, class T>
inline bool empty(HashMap<K, T> const& hm) { return size(hm) == 0; }

/// Beginning iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T>* begin(HashMap<K, T>& hm) {
	return array::begin(hm._data);
}
/// Beginning iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T> const* begin(HashMap<K, T> const& hm) {
	return array::begin(hm._data);
}

/// Ending iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T>* end(HashMap<K, T>& hm) {
	return array::end(hm._data);
}
/// Ending iterator: [begin, end).
template<class K, class T>
inline HashMapNode<K, T> const* end(HashMap<K, T> const& hm) {
	return array::end(hm._data);
}

/** @cond INTERNAL */
enum : unsigned {
	END = ~0u,
};

struct FindData {
	// Index into _head
	u32 head;
	// Index into _data
	u32 data;
	// Previous node with same key
	u32 data_prev;
};

namespace internal {

template<class K, class T>
FindData find(HashMap<K, T> const& hm, K const key) {
	FindData fd{END, END, END};
	if (array::empty(hm._head)) {
		return fd;
	}

	fd.head = key % num_partitions(hm);
	fd.data = hm._head[fd.head];
	while (fd.data != END) {
		auto const& node = hm._data[fd.data];
		if (node.key == key) {
			break;
		}
		fd.data_prev = fd.data;
		fd.data = node.next;
	}
	return fd;
}

template<class K, class T>
FindData find(
	HashMap<K, T> const& hm,
	HashMapNode<K, T> const* node
) {
	FindData fd{END, END, END};
	if (array::empty(hm._head)) {
		return fd;
	}

	fd.head = node->key % num_partitions(hm);
	fd.data = hm._head[fd.head];
	while (fd.data != END) {
		auto const& it_node = hm._data[fd.data];
		if (&it_node == node) {
			break;
		}
		fd.data_prev = fd.data;
		fd.data = it_node.next;
	}
	return fd;
}

template<class K, class T>
u32 make(HashMap<K, T>& hm, K const key, bool const keep) {
	TOGO_DEBUG_ASSERTE(array::any(hm._head));
	FindData const fd = find(hm, key);
	if (keep && fd.data != END) {
		return fd.data;
	}
	u32 const index = array::size(hm._data);
	HashMapNode<K, T> node;
	node.key = key;
	node.next = fd.data;
	array::push_back(hm._data, node);
	if (fd.data_prev == END) {
		hm._head[fd.head] = index;
	} else {
		hm._data[fd.data_prev].next = index;
	}
	return index;
}

template<class K, class T>
void remove(HashMap<K, T>& hm, FindData const& fd) {
	if (fd.data_prev == END) {
		// find() was immediate hit; fd is the first item
		// in the head. Move the head to the next item (if any).
		hm._head[fd.head] = hm._data[fd.data].next;
	} else {
		hm._data[fd.data_prev].next = hm._data[fd.data].next;
	}

	// Move last node to the removed position
	if (fd.data != array::size(hm._data) - 1) {
		auto const& last_node = array::back(hm._data);
		FindData const last = find(hm, &last_node);
		if (last.data_prev == END) {
			hm._head[last.head] = fd.data;
		} else {
			hm._data[last.data_prev].next = fd.data;
		}
		hm._data[fd.data] = last_node;
	}
	array::pop_back(hm._data);
}

template<class K, class T>
void resize(HashMap<K, T>& hm, u32_fast const new_size) {
	HashMap<K, T> new_hm{*hm._head._allocator};
	array::resize(new_hm._head, new_size);
	array::reserve(new_hm._data, min(new_size, array::capacity(hm._data)));
	for (unsigned i = 0; i < new_size; ++i) {
		new_hm._head[i] = END;
	}
	u32 index;
	for (auto const& node : hm._data) {
		index = make(hm, node.key, false);
		hm._data[index].value = node.value;
	}
	hm = std::move(new_hm);
}

template<class K, class T>
inline void grow(HashMap<K, T>& hm) {
	resize(hm, num_partitions(hm) * 2 + 16);
}

} // namespace internal
/** @endcond */ // INTERNAL

/// Reserve at least new_capacity.
template<class K, class T>
inline void reserve(HashMap<K, T>& hm, u32_fast const new_capacity) {
	if (new_capacity > capacity(hm)) {
		internal::resize(hm, (new_capacity / TOGO_HASH_MAP_MAX_LOAD) + 1);
	}
}

/// Remove all items.
template<class K, class T>
inline void clear(HashMap<K, T>& hm) {
	auto const size = num_partitions(hm);
	for (unsigned i = 0; i < size; ++i) {
		hm._head[i] = END;
	}
	array::clear(hm._data);
}

/// Set item.
///
/// If key does not exist, it will be inserted.
template<class K, class T>
inline void set(HashMap<K, T>& hm, K const key, T const& value) {
	if (!space(hm)) {
		internal::grow(hm);
	}
	auto const index = internal::make(hm, key, true);
	hm._data[index].value = value;
}

/// Push item.
///
/// If there are existing values with key, they are retained.
template<class K, class T>
inline void push(HashMap<K, T>& hm, K const key, T const& value) {
	if (!space(hm)) {
		internal::grow(hm);
	}
	auto const index = internal::make(hm, key, false);
	hm._data[index].value = value;
}

/// Get item.
///
/// If key does not exist, nullptr will be returned.
/// If multiple values are assigned to key, this will get the most
/// recently pushed one.
template<class K, class T>
inline T* get(HashMap<K, T>& hm, K const key) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index].value : nullptr;
}

/// Get item.
///
/// If key does not exist, nullptr will be returned.
/// If multiple values are assigned to key, this will get the most
/// recently pushed one.
template<class K, class T>
inline T const* get(HashMap<K, T> const& hm, K const key) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index].value : nullptr;
}

/// Get first node with key.
///
/// If there are no items with key, nullptr will be returned.
template<class K, class T>
inline HashMapNode<K, T>* get_node(
	HashMap<K, T>& hm,
	K const key
) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index] : nullptr;
}

/// Get first node with key.
///
/// If there are no items with key, nullptr will be returned.
template<class K, class T>
inline HashMapNode<K, T> const* get_node(
	HashMap<K, T> const& hm,
	K const key
) {
	auto const index = internal::find(hm, key).data;
	return (index != END) ? &hm._data[index] : nullptr;
}

/// Get next node in keyset.
///
/// An assertion will fail if node is nullptr. Returns nullptr when
/// there are no more nodes for the key.
template<class K, class T>
inline HashMapNode<K, T>* get_next(
	HashMap<K, T>& hm,
	HashMapNode<K, T>* node
) {
	TOGO_ASSERTE(node != nullptr);
	K const key = node->key;
	while (node->next != END) {
		node = &hm._data[node->next];
		if (node->key == key) {
			return node;
		}
	}
	return nullptr;
}

/// Get next node in keyset.
///
/// An assertion will fail if node is nullptr. Returns nullptr when
/// there are no more nodes for the key.
template<class K, class T>
inline HashMapNode<K, T> const* get_next(
	HashMap<K, T> const& hm,
	HashMapNode<K, T> const* node
) {
	TOGO_ASSERTE(node != nullptr);
	K const key = node->key;
	while (node->next != END) {
		node = &hm._data[node->next];
		if (node->key == key) {
			return node;
		}
	}
	return nullptr;
}

/// Check if there is a value with key.
template<class K, class T>
inline bool has(HashMap<K, T> const& hm, K const key) {
	return internal::find(hm, key).data != END;
}

/// Count the number of values with key.
template<class K, class T>
inline unsigned count(HashMap<K, T> const& hm, K const key) {
	auto const* node = get_node(hm, key);
	unsigned count = 0;
	while (node != nullptr) {
		node = get_next(hm, node);
		++count;
	}
	return count;
}

/// Remove value.
///
/// If there are multiple values with key, the most recently added
/// one is removed.
template<class K, class T>
inline void remove(HashMap<K, T>& hm, K const key) {
	FindData const fd = internal::find(hm, key);
	if (fd.data != END) {
		internal::remove(hm, fd);
	}
}

/// Remove node.
///
/// An assertion will fail if node is nullptr.
template<class K, class T>
inline void remove(HashMap<K, T>& hm, HashMapNode<K, T> const* node) {
	TOGO_ASSERTE(node != nullptr);
	FindData const fd = internal::find(hm, node);
	if (fd.data != END) {
		internal::remove(hm, fd);
	}
}

/** @cond INTERNAL */
#undef TOGO_HASH_MAP_MAX_LOAD
/** @endcond */ // INTERNAL

/** @} */ // end of doc-group hash_map

} // namespace hash_map

/** @cond INTERNAL */

// ADL support

template<class K, class T>
inline HashMapNode<K, T>* begin(HashMap<K, T>& hm) {
	return hash_map::begin(hm);
}
template<class K, class T>
inline HashMapNode<K, T> const* begin(HashMap<K, T> const& hm) {
	return hash_map::begin(hm);
}

template<class K, class T>
inline HashMapNode<K, T>* end(HashMap<K, T>& hm) {
	return hash_map::end(hm);
}
template<class K, class T>
inline HashMapNode<K, T> const* end(HashMap<K, T> const& hm) {
	return hash_map::end(hm);
}

/** @endcond */ // INTERNAL

} // namespace togo
<|endoftext|>


<commit_before>/*
 * Copyright 2007-2017 Content Management AG
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "serialize.hxx"
#include "strmap.hxx"
#include "GrowingBuffer.hxx"
#include "util/ConstBuffer.hxx"
#include "util/ByteOrder.hxx"

#include <assert.h>
#include <stdint.h>
#include <string.h>

void
serialize_uint16(GrowingBuffer &gb, uint16_t value)
{
    uint16_t *dest = (uint16_t *)gb.Write(sizeof(*dest));
    *dest = ToBE16(value);
}

void
serialize_uint32(GrowingBuffer &gb, uint32_t value)
{
    uint32_t *dest = (uint32_t *)gb.Write(sizeof(*dest));
    *dest = ToBE32(value);
}

void
serialize_uint64(GrowingBuffer &gb, uint64_t value)
{
    uint64_t *dest = (uint64_t *)gb.Write(sizeof(*dest));
    *dest = ToBE64(value);
}

/*
static void
serialize_size_t(GrowingBuffer &gb, size_t value)
{
    serialize_uint32(gb, value);
}
*/

void
serialize_string(GrowingBuffer &gb, const char *value)
{
    assert(value != nullptr);

    /* write the string including the null terminator */
    gb.Write(value, strlen(value) + 1);
}

void
serialize_string_null(GrowingBuffer &gb, const char *value)
{
    serialize_string(gb, value != nullptr ? value : "");
}

void
serialize_strmap(GrowingBuffer &gb, const StringMap &map)
{
    for (const auto &i : map) {
        if (*i.key == 0)
            /* this shouldn't happen; ignore this invalid entry  */
            continue;

        serialize_string(gb, i.key);
        serialize_string(gb, i.value);
    }

    /* key length 0 means "end of map" */
    serialize_string(gb, "");
}

void
serialize_strmap(GrowingBuffer &gb, const StringMap *map)
{
    if (map == nullptr)
        /* same as empty map */
        serialize_string(gb, "");
    else
        serialize_strmap(gb, *map);
}

static void
SkipFront(ConstBuffer<void> &input, size_t n)
{
    assert(input.size >= n);

    input.data = (const uint8_t *)input.data + n;
    input.size -= n;
}

template<typename T>
static void
DeserializeT(ConstBuffer<void> &input, T &dest)
{
    static_assert(std::is_trivial<T>::value, "type is not trivial");

    if (gcc_unlikely(input.size < sizeof(dest)))
        throw DeserializeError();

    memcpy(&dest, input.data, sizeof(dest));
    SkipFront(input, sizeof(dest));
}

uint16_t
deserialize_uint16(ConstBuffer<void> &input)
{
    uint16_t value;
    DeserializeT(input, value);
    return FromBE16(*(const uint16_t *)input.data);
}

uint32_t
deserialize_uint32(ConstBuffer<void> &input)
{
    uint32_t value;
    DeserializeT(input, value);
    return FromBE32(*(const uint32_t *)input.data);
}

uint64_t
deserialize_uint64(ConstBuffer<void> &input)
{
    uint64_t value;
    DeserializeT(input, value);
    return FromBE64(*(const uint64_t *)input.data);
}

const char *
deserialize_string(ConstBuffer<void> &input)
{
    const char *end = (const char *)memchr(input.data, 0, input.size);
    if (end == nullptr)
        throw DeserializeError();

    const char *value = (const char *)input.data;

    SkipFront(input, end + 1 - value);
    return value;
}

const char *
deserialize_string_null(ConstBuffer<void> &input)
{
    const char *value = deserialize_string(input);
    if (*value == 0)
        value = nullptr;
    return value;
}

void
deserialize_strmap(ConstBuffer<void> &input, StringMap &dest)
{
    while (true) {
        const char *key = deserialize_string(input);
        if (*key == 0)
            break;

        const char *value = deserialize_string(input);

        dest.Add(key, value);
    }
}
<commit_msg>serialize: fix FromBE*() parameters<commit_after>/*
 * Copyright 2007-2017 Content Management AG
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "serialize.hxx"
#include "strmap.hxx"
#include "GrowingBuffer.hxx"
#include "util/ConstBuffer.hxx"
#include "util/ByteOrder.hxx"

#include <assert.h>
#include <stdint.h>
#include <string.h>

void
serialize_uint16(GrowingBuffer &gb, uint16_t value)
{
    uint16_t *dest = (uint16_t *)gb.Write(sizeof(*dest));
    *dest = ToBE16(value);
}

void
serialize_uint32(GrowingBuffer &gb, uint32_t value)
{
    uint32_t *dest = (uint32_t *)gb.Write(sizeof(*dest));
    *dest = ToBE32(value);
}

void
serialize_uint64(GrowingBuffer &gb, uint64_t value)
{
    uint64_t *dest = (uint64_t *)gb.Write(sizeof(*dest));
    *dest = ToBE64(value);
}

/*
static void
serialize_size_t(GrowingBuffer &gb, size_t value)
{
    serialize_uint32(gb, value);
}
*/

void
serialize_string(GrowingBuffer &gb, const char *value)
{
    assert(value != nullptr);

    /* write the string including the null terminator */
    gb.Write(value, strlen(value) + 1);
}

void
serialize_string_null(GrowingBuffer &gb, const char *value)
{
    serialize_string(gb, value != nullptr ? value : "");
}

void
serialize_strmap(GrowingBuffer &gb, const StringMap &map)
{
    for (const auto &i : map) {
        if (*i.key == 0)
            /* this shouldn't happen; ignore this invalid entry  */
            continue;

        serialize_string(gb, i.key);
        serialize_string(gb, i.value);
    }

    /* key length 0 means "end of map" */
    serialize_string(gb, "");
}

void
serialize_strmap(GrowingBuffer &gb, const StringMap *map)
{
    if (map == nullptr)
        /* same as empty map */
        serialize_string(gb, "");
    else
        serialize_strmap(gb, *map);
}

static void
SkipFront(ConstBuffer<void> &input, size_t n)
{
    assert(input.size >= n);

    input.data = (const uint8_t *)input.data + n;
    input.size -= n;
}

template<typename T>
static void
DeserializeT(ConstBuffer<void> &input, T &dest)
{
    static_assert(std::is_trivial<T>::value, "type is not trivial");

    if (gcc_unlikely(input.size < sizeof(dest)))
        throw DeserializeError();

    memcpy(&dest, input.data, sizeof(dest));
    SkipFront(input, sizeof(dest));
}

uint16_t
deserialize_uint16(ConstBuffer<void> &input)
{
    uint16_t value;
    DeserializeT(input, value);
    return FromBE16(value);
}

uint32_t
deserialize_uint32(ConstBuffer<void> &input)
{
    uint32_t value;
    DeserializeT(input, value);
    return FromBE32(value);
}

uint64_t
deserialize_uint64(ConstBuffer<void> &input)
{
    uint64_t value;
    DeserializeT(input, value);
    return FromBE64(value);
}

const char *
deserialize_string(ConstBuffer<void> &input)
{
    const char *end = (const char *)memchr(input.data, 0, input.size);
    if (end == nullptr)
        throw DeserializeError();

    const char *value = (const char *)input.data;

    SkipFront(input, end + 1 - value);
    return value;
}

const char *
deserialize_string_null(ConstBuffer<void> &input)
{
    const char *value = deserialize_string(input);
    if (*value == 0)
        value = nullptr;
    return value;
}

void
deserialize_strmap(ConstBuffer<void> &input, StringMap &dest)
{
    while (true) {
        const char *key = deserialize_string(input);
        if (*key == 0)
            break;

        const char *value = deserialize_string(input);

        dest.Add(key, value);
    }
}
<|endoftext|>


<commit_before>#include <set.hpp>

#include <kdb.hpp>
#include <kdbio.hpp>
#include <cmdline.hpp>

#include <iostream>

using namespace std;
using namespace kdb;

SetCommand::SetCommand()
{}

int SetCommand::execute(Cmdline const& cl)
{
	int argc = cl.arguments.size();
	if (argc != 1 && argc != 2)
	{
		throw invalid_argument("1 or 2 arguments needed");
	}

	std::string name = cl.arguments[0];

	bool nullValue;
	std::string value;

	if (argc == 2)
	{
		nullValue = false;
		value = cl.arguments[1];
	}
	else
	{
		nullValue = true;
	}

	KeySet conf;
	Key k(name, KEY_END);

	// do not resume on any get errors
	// otherwise the user might break
	// the config
	kdb.get(conf, k);

	printWarnings(cerr, k);
	printError(cerr, k);

	Key key = conf.lookup(name);

	if (!key)
	{
		cout << "create a new key " << name;
		key = Key(name, KEY_END);
		if (!nullValue)
		{
			cout << " with string " << value << endl;
			key.setString(value);
		} else {
			cout << " with null value" << endl;
			key.setBinary(0, 0);
		}
		if (!key.isValid())
		{
			cerr << "no valid name supplied" << endl;
			return 1;
		}
		conf.append(key);
	} else {
		if (!nullValue)
		{
			cout << "Set string to " << value << endl;
			key.setString(value);
		} else {
			cout << "Set null value" << endl;
			key.setBinary(0, 0);
		}
	}
	Key n;
	kdb.set(conf, n);
	printWarnings(cerr, n);
	printError(cerr, n);

	return 0;
}

SetCommand::~SetCommand()
{}
<commit_msg>use cascading name for kdb set<commit_after>#include <set.hpp>

#include <kdb.hpp>
#include <kdbio.hpp>
#include <cmdline.hpp>

#include <iostream>

using namespace std;
using namespace kdb;

SetCommand::SetCommand()
{}

int SetCommand::execute(Cmdline const& cl)
{
	int argc = cl.arguments.size();
	if (argc != 1 && argc != 2)
	{
		throw invalid_argument("1 or 2 arguments needed");
	}

	std::string name = cl.arguments[0];

	bool nullValue;
	std::string value;

	if (argc == 2)
	{
		nullValue = false;
		value = cl.arguments[1];
	}
	else
	{
		nullValue = true;
	}

	KeySet conf;
	Key k(name, KEY_END);

	// do not resume on any get errors
	// otherwise the user might break
	// the config
	kdb.get(conf, k);

	printWarnings(cerr, k);
	printError(cerr, k);

	Key key = conf.lookup(name);

	if (!key)
	{
		cout << "create a new key " << name;
		key = Key(name, KEY_END);
		if (!nullValue)
		{
			cout << " with string " << value << endl;
			key.setString(value);
		} else {
			cout << " with null value" << endl;
			key.setBinary(0, 0);
		}
		if (!key.isValid())
		{
			cerr << "no valid name supplied" << endl;
			return 1;
		}
		conf.append(key);
	} else {
		if (!nullValue)
		{
			cout << "Set string to " << value << endl;
			key.setString(value);
		} else {
			cout << "Set null value" << endl;
			key.setBinary(0, 0);
		}
	}
	Key n("/", KEY_CASCADING_NAME, KEY_END);
	kdb.set(conf, n);
	printWarnings(cerr, n);
	printError(cerr, n);

	return 0;
}

SetCommand::~SetCommand()
{}
<|endoftext|>


<commit_before>#include "tracker.h"

#include "ada/tools/text.h"

void Tracker::start() {
    m_data.clear();

    auto start = std::chrono::high_resolution_clock::now();
    m_trackerStart = std::chrono::time_point_cast<std::chrono::microseconds>(start).time_since_epoch().count() * 0.001;
    m_running = true;
}

void Tracker::begin(const std::string& _track) {
    if (!m_running)
        return;

    m_stack.push_back(_track);
    std::string stack = getStack();

    if ( m_data.find(stack) == m_data.end() )
        m_tracks.push_back(stack);

    m_data[stack].start = std::chrono::high_resolution_clock::now();
}

void Tracker::end(const std::string& _track) {
    if (!m_running)
        return;

    auto sample_end = std::chrono::high_resolution_clock::now();

    std::string stack = getStack();
    m_stack.pop_back();

    if ( m_data.find(stack) == m_data.end() )
        m_tracks.push_back(stack);

    auto start = std::chrono::time_point_cast<std::chrono::microseconds>(m_data[stack].start).time_since_epoch();
    auto end = std::chrono::time_point_cast<std::chrono::microseconds>(sample_end).time_since_epoch();

    StatSample stat;
    stat.startMs = start.count() * 0.001 - m_trackerStart;
    stat.endMs = end.count() * 0.001 - m_trackerStart;
    stat.durationMs = stat.endMs - stat.startMs;

    m_data[stack].samples.push_back( stat );
}

void Tracker::stop() {
    m_running = false;
}

double  Tracker::getFramerate() {
    double frm = 0.0;
    int count = 0;
    for (std::map<std::string, StatTrack>::iterator it = m_data.begin() ; it != m_data.end(); ++it) {
        double delta = 0.0;
        for (size_t i = 1; i < it->second.samples.size(); i++)
            delta += (it->second.samples[i].startMs - it->second.samples[i-1].startMs);

        delta /= (double) (it->second.samples.size() - 1);
        frm += delta;
        count++;
    }

    return ( frm / (double)count );
}

std::string Tracker::getStack() const {
    std::string stack = "";

    for (size_t i = 0; i < m_stack.size(); i++) {
        if (i > 0)
            stack += ":";
        stack += m_stack[i];
    }

    return stack;
}

std::string Tracker::logFramerate() {
    return  "framerate," + ada::toString(getFramerate()) + ",100.0%\n";// +
            // "fps," + ada::toString( (1./getFramerate()) * 1000.0 ) ;
}

std::string Tracker::logSamples() {
    std::string log = "";

    for (size_t t = 0; t < m_tracks.size(); t++)
        log += logSamples(m_tracks[t]);

    return log;
}

std::string Tracker::logSamples(const std::string& _track) {
    std::map<std::string, StatTrack>::iterator it = m_data.find(_track);

    if ( it == m_data.end() )
        return "";

    std::string log = "";
    std::string track_name = it->first;

    for (size_t i = 0; i < it->second.samples.size(); i++)
        log +=  track_name + "," +
                ada::toString(it->second.samples[i].startMs) + "," +
                ada::toString(it->second.samples[i].durationMs) + "\n";

    return log;
}

std::string Tracker::logAverage() {
    std::string log = "";

    for (size_t t = 0; t < m_tracks.size(); t++)
        log += logAverage(m_tracks[t]);

    return log;
}

std::string Tracker::logAverage(const std::string& _track) {
    std::map<std::string, StatTrack>::iterator it = m_data.find(_track);

    if ( it == m_data.end() )
        return "";

    std::string log = "";
    std::string track_name = it->first;

    double average = 0.0;
    double delta = 0.0;
    for (size_t i = 0; i < it->second.samples.size(); i++) {
        average += it->second.samples[i].durationMs;
        if (i > 0)
            delta += it->second.samples[i].startMs - it->second.samples[i-1].startMs;
    }

    average /= (double)it->second.samples.size();
    delta /= (double)it->second.samples.size() - 1.0;
    it->second.durationAverage = average;

    log += track_name + "," + ada::toString(average) + "," + ada::toString( (average/delta) * 100.0) + "%," + ada::toString(delta) +  "\n";

    return log;
}
<commit_msg>remove pct<commit_after>#include "tracker.h"

#include "ada/tools/text.h"

void Tracker::start() {
    m_data.clear();

    auto start = std::chrono::high_resolution_clock::now();
    m_trackerStart = std::chrono::time_point_cast<std::chrono::microseconds>(start).time_since_epoch().count() * 0.001;
    m_running = true;
}

void Tracker::begin(const std::string& _track) {
    if (!m_running)
        return;

    m_stack.push_back(_track);
    std::string stack = getStack();

    if ( m_data.find(stack) == m_data.end() )
        m_tracks.push_back(stack);

    m_data[stack].start = std::chrono::high_resolution_clock::now();
}

void Tracker::end(const std::string& _track) {
    if (!m_running)
        return;

    auto sample_end = std::chrono::high_resolution_clock::now();

    std::string stack = getStack();
    m_stack.pop_back();

    if ( m_data.find(stack) == m_data.end() )
        m_tracks.push_back(stack);

    auto start = std::chrono::time_point_cast<std::chrono::microseconds>(m_data[stack].start).time_since_epoch();
    auto end = std::chrono::time_point_cast<std::chrono::microseconds>(sample_end).time_since_epoch();

    StatSample stat;
    stat.startMs = start.count() * 0.001 - m_trackerStart;
    stat.endMs = end.count() * 0.001 - m_trackerStart;
    stat.durationMs = stat.endMs - stat.startMs;

    m_data[stack].samples.push_back( stat );
}

void Tracker::stop() {
    m_running = false;
}

double  Tracker::getFramerate() {
    double frm = 0.0;
    int count = 0;
    for (std::map<std::string, StatTrack>::iterator it = m_data.begin() ; it != m_data.end(); ++it) {
        double delta = 0.0;
        for (size_t i = 1; i < it->second.samples.size(); i++)
            delta += (it->second.samples[i].startMs - it->second.samples[i-1].startMs);

        delta /= (double) (it->second.samples.size() - 1);
        frm += delta;
        count++;
    }

    return ( frm / (double)count );
}

std::string Tracker::getStack() const {
    std::string stack = "";

    for (size_t i = 0; i < m_stack.size(); i++) {
        if (i > 0)
            stack += ":";
        stack += m_stack[i];
    }

    return stack;
}

std::string Tracker::logFramerate() {
    return  "framerate," + ada::toString(getFramerate()) + "\n";// +
            // "fps," + ada::toString( (1./getFramerate()) * 1000.0 ) ;
}

std::string Tracker::logSamples() {
    std::string log = "";

    for (size_t t = 0; t < m_tracks.size(); t++)
        log += logSamples(m_tracks[t]);

    return log;
}

std::string Tracker::logSamples(const std::string& _track) {
    std::map<std::string, StatTrack>::iterator it = m_data.find(_track);

    if ( it == m_data.end() )
        return "";

    std::string log = "";
    std::string track_name = it->first;

    for (size_t i = 0; i < it->second.samples.size(); i++)
        log +=  track_name + "," +
                ada::toString(it->second.samples[i].startMs) + "," +
                ada::toString(it->second.samples[i].durationMs) + "\n";

    return log;
}

std::string Tracker::logAverage() {
    std::string log = "";

    for (size_t t = 0; t < m_tracks.size(); t++)
        log += logAverage(m_tracks[t]);

    return log;
}

std::string Tracker::logAverage(const std::string& _track) {
    std::map<std::string, StatTrack>::iterator it = m_data.find(_track);

    if ( it == m_data.end() )
        return "";

    std::string log = "";
    std::string track_name = it->first;

    double average = 0.0;
    double delta = 0.0;
    for (size_t i = 0; i < it->second.samples.size(); i++) {
        average += it->second.samples[i].durationMs;
        if (i > 0)
            delta += it->second.samples[i].startMs - it->second.samples[i-1].startMs;
    }

    average /= (double)it->second.samples.size();
    delta /= (double)it->second.samples.size() - 1.0;
    it->second.durationAverage = average;

    log += track_name + "," + ada::toString(average) + "," + ada::toString( (average/delta) * 100.0) + "%," + ada::toString(delta) +  "\n";

    return log;
}
<|endoftext|>


<commit_before>#include "training_core.hpp"
#include "nanopolish_emissions.h"
#include "logsumset.hpp"
#include "logger.hpp"

using std::string;
using std::vector;
using std::multiset;
using std::endl;

const bool use_multiset_logsum =
#ifndef USE_MULTISET_LOGSUM
    false;
#else
    true;
#endif

GaussianMixture train_gaussian_mixture(const vector< StateTrainingData >& data, const GaussianMixture& input_mixture)
{

    size_t n_components = input_mixture.params.size();
    size_t n_data = data.size();
    float log_n_data = std::log(n_data);
    assert(input_mixture.log_weights.size() == n_components);
    GaussianMixture curr_mixture = input_mixture;

    for(size_t iteration = 0; iteration < 10; ++iteration) {
        GaussianMixture new_mixture = curr_mixture;

        // compute log_pdfs
        //
        //   pdf[i][j] := gauss(mu_j, sigma_j * read_var_i, level_mean_i)
        //
        vector< vector< float > > log_pdf(n_data);
        for(size_t i = 0; i < n_data; ++i) {
            log_pdf[i].resize(n_components);
            for(size_t j = 0; j < n_components; ++j) {
                // We need to scale the mixture component parameters by the per-read var factor
                PoreModelStateParams scaled_state = curr_mixture.params[j];
                scaled_state.level_stdv *= data[i].read_var;
                scaled_state.level_log_stdv += data[i].log_read_var;
                log_pdf[i][j] = log_normal_pdf(data[i].level_mean, scaled_state);
                assert(not std::isnan(log_pdf[i][j]));
            }
        }

        // compute responsibilities
        //
        //   resp[i][j] := ( w_j * pdf[i][j] ) / sum_k ( w_k * pdf[i][k] )
        //
        vector< vector< float > > log_resp(n_data);
        for(size_t i = 0; i < n_data; ++i) {
            log_resp[i].resize(n_components);
            logsumset< float > denom_terms(use_multiset_logsum);
            for(size_t j = 0; j < n_components; ++j) {
                float v = log_pdf[i][j] + curr_mixture.log_weights[j];
                log_resp[i][j] = v;
                denom_terms.add(v);
            }
            float log_denom = denom_terms.val();
            for(size_t j = 0; j < n_components; ++j) {
                log_resp[i][j] -= log_denom;
            }
        }

        // update weights
        //
        //   w'[j] := sum_i resp[i][j] / n_data
        //
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                numer_terms.add(log_resp[i][j]);
            }
            float log_numer = numer_terms.val();
            new_mixture.log_weights[j] = log_numer - log_n_data;
        }

        // update means
        //
        //   mu_j := sum_i ( resp[i][j] * level_mean_i ) / sum_i resp[i][j]
        //         = sum_i ( resp[i][j] * level_mean_i ) / ( w'[j] * n_data )
        //
        vector< float > new_log_mean(2);
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                numer_terms.add(log_resp[i][j] + data[i].log_level_mean);
            }
            float log_numer = numer_terms.val();
            new_log_mean[j] = log_numer - (log_n_data + new_mixture.log_weights[j]);
        }

        // update stdvs
        //
        //   var_j := sum_i ( resp[i][j] * ( ( level_mean_i - mu_j ) / read_var_i )^2 ) / sum_i resp[i][j]
        //          = sum_i ( resp[i][j] * ( ( level_mean_i - mu_j ) / read_var_i )^2 ) / ( w'[j] * n_data )
        //
        vector< float > new_log_var(2);
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                float v = std::abs(data[i].level_mean - std::exp(new_log_mean[j]));
                numer_terms.add(log_resp[i][j] + (not std::isnan(v) and v > 0? 2.0 * (std::log(v) - data[i].log_read_var) : 0.0));
            }
            float log_numer = numer_terms.val();
            new_log_var[j] = log_numer - (log_n_data + new_mixture.log_weights[j]);
        }

        for(size_t j = 0; j < n_components; ++j) {
            new_mixture.params[j].level_mean = std::exp(new_log_mean[j]);
            new_mixture.params[j].level_log_stdv = .5 * new_log_var[j];
            new_mixture.params[j].level_stdv = std::exp(new_mixture.params[j].level_log_stdv);
            LOG("training_core", info)
                << "new_mixture " << iteration << " " << j << " "
                << std::fixed << std::setprecision(2) << std::exp(new_mixture.log_weights[j]) << " "
                << new_mixture.params[j].level_mean << " " << new_mixture.params[j].level_stdv << endl;
        }

        curr_mixture = new_mixture;
    }
    return curr_mixture;
}

InvGaussianMixture train_invgaussian_mixture(const vector< StateTrainingData >& data, const InvGaussianMixture& in_mixture)
{
    size_t n_components = in_mixture.params.size();
    assert(in_mixture.log_weights.size() == n_components);
    size_t n_data = data.size();
    auto crt_mixture = in_mixture;

    // compute gaussian pdfs
    //
    //   pdf[i][j].first = gauss(mu_j, sigma_j * read_var_i, level_mean_i)
    //
    vector< vector< std::pair< float, float > > > log_pdf(n_data);
    for (size_t i = 0; i < n_data; ++i) {
        log_pdf[i].resize(n_components);
        for (size_t j = 0; j < n_components; ++j) {
            PoreModelStateParams scaled_state = in_mixture.params[j];
            scaled_state.level_stdv *= data[i].read_var;
            scaled_state.level_log_stdv += data[i].log_read_var;
            log_pdf[i][j].first = log_normal_pdf(data[i].level_mean, scaled_state);
            assert(not std::isnan(log_pdf[i][j].first));
            LOG("training_core", debug)
                << "log_gauss_pdf " << i << " " << j << " " << std::scientific << log_pdf[i][j].first << endl;
        }
    }

    // compute gaussian weights
    //
    //   g_weights[i][j] := ( w_j * pdf[i][j].first ) / sum_k ( w_k * pdf[i][k].first )
    //
    vector< vector< float > > log_g_weights(n_data);
    for (size_t i = 0; i < n_data; ++i) {
        log_g_weights[i].resize(n_components);
        logsumset< float > denom_terms(use_multiset_logsum);
        for (size_t j = 0; j < n_components; ++j) {
            float v = in_mixture.log_weights[j] + log_pdf[i][j].first;
            log_g_weights[i][j] = v;
            denom_terms.add(v);
        }
        float log_denom = denom_terms.val();
        for (size_t j = 0; j < n_components; ++j) {
            log_g_weights[i][j] -= log_denom;
            LOG("training_core", debug)
                << "g_weights " << i << " " << j << " " << std::scientific << std::exp(log_g_weights[i][j]) << endl;
        }
    }

    for (size_t iteration = 0; iteration < 10; ++iteration) {
        // compute inverse gaussian pdfs
        //
        //   pdf[i][j].second = invgauss(eta_j, lambda_j * ( read_var_sd_i / read_var_scale_i ), level_stdv_i)
        //
        for (size_t i = 0; i < n_data; ++i) {
            for (size_t j = 0; j < n_components; ++j) {
                PoreModelStateParams scaled_state = in_mixture.params[j];
                scaled_state.sd_lambda *= data[i].read_var_sd / data[i].read_scale_sd;
                scaled_state.sd_log_lambda += data[i].log_read_var_sd - data[i].log_read_scale_sd;
                log_pdf[i][j].second = log_invgauss_pdf(data[i].level_stdv, data[i].log_level_stdv, scaled_state);
                assert(not std::isnan(log_pdf[i][j].second));
                LOG("training_core", debug)
                    << "log_invgauss_pdf " << i << " " << j << " " << std::scientific << log_pdf[i][j].second << endl;
            }
        }
        // compute inverse gaussian weights (responsibilities)
        //
        //   ig_weights[i][j] := ( g_weights[i][j] * pdf[i][j].second ) / sum_k ( g_weights[i][k] * pdf[i][k].second )
        //
        vector< vector< float > > log_ig_weights(n_data);
        for (size_t i = 0; i < n_data; ++i) {
            log_ig_weights[i].resize(n_components);
            logsumset< float > denom_terms(use_multiset_logsum);
            for (size_t j = 0; j < n_components; ++j) {
                float v = log_g_weights[i][j] + log_pdf[i][j].second;
                log_ig_weights[i][j] = v;
                denom_terms.add(v);
            }
            float log_denom = denom_terms.val();
            for (size_t j = 0; j < n_components; ++j) {
                log_ig_weights[i][j] -= log_denom;
                LOG("training_core", debug)
                    << "ig_weights " << i << " " << j << " " << std::scientific << std::exp(log_ig_weights[i][j]) << endl;
            }
        }

        // update eta
        //
        //   eta_j := sum_i ( ig_weigts[i][j] * lambda'_ij * level_stdv_i ) / sum_i ( ig_weights[i][j] * lambda'_ij )
        //   lambda'_ij := lambda_j * ( read_var_sd_i / read_var_scale_i )
        //
        auto new_mixture = crt_mixture;
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            logsumset< float > denom_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                float v = log_ig_weights[i][j] + in_mixture.params[j].sd_log_lambda + (data[i].log_read_var_sd - data[i].log_read_scale_sd);
                numer_terms.add(v + data[i].log_level_stdv);
                denom_terms.add(v);
            }
            float log_numer = numer_terms.val();
            float log_denom = denom_terms.val();
            new_mixture.params[j].sd_mean = std::exp(log_numer - log_denom);
            LOG("training_core", info)
                << "new_mixture " << iteration << " " << j << " "
                << std::fixed << std::setprecision(2) << std::exp(new_mixture.log_weights[j]) << " "
                << new_mixture.params[j].sd_mean << endl;
        }
        std::swap(crt_mixture, new_mixture);
    } // for iteration

    return crt_mixture;
} // train_ig_mixture
<commit_msg>log values in normal space, not logspace<commit_after>#include "training_core.hpp"
#include "nanopolish_emissions.h"
#include "logsumset.hpp"
#include "logger.hpp"

using std::string;
using std::vector;
using std::multiset;
using std::endl;

const bool use_multiset_logsum =
#ifndef USE_MULTISET_LOGSUM
    false;
#else
    true;
#endif

GaussianMixture train_gaussian_mixture(const vector< StateTrainingData >& data, const GaussianMixture& input_mixture)
{

    size_t n_components = input_mixture.params.size();
    size_t n_data = data.size();
    float log_n_data = std::log(n_data);
    assert(input_mixture.log_weights.size() == n_components);
    GaussianMixture curr_mixture = input_mixture;

    for(size_t iteration = 0; iteration < 10; ++iteration) {
        GaussianMixture new_mixture = curr_mixture;

        // compute log_pdfs
        //
        //   pdf[i][j] := gauss(mu_j, sigma_j * read_var_i, level_mean_i)
        //
        vector< vector< float > > log_pdf(n_data);
        for(size_t i = 0; i < n_data; ++i) {
            log_pdf[i].resize(n_components);
            for(size_t j = 0; j < n_components; ++j) {
                // We need to scale the mixture component parameters by the per-read var factor
                PoreModelStateParams scaled_state = curr_mixture.params[j];
                scaled_state.level_stdv *= data[i].read_var;
                scaled_state.level_log_stdv += data[i].log_read_var;
                log_pdf[i][j] = log_normal_pdf(data[i].level_mean, scaled_state);
                assert(not std::isnan(log_pdf[i][j]));
                LOG("training_core", debug)
                    << "pdf " << i << " " << j << " "
                    << std::scientific << std::exp(log_pdf[i][j]) << endl;
            }
        }

        // compute responsibilities
        //
        //   resp[i][j] := ( w_j * pdf[i][j] ) / sum_k ( w_k * pdf[i][k] )
        //
        vector< vector< float > > log_resp(n_data);
        for(size_t i = 0; i < n_data; ++i) {
            log_resp[i].resize(n_components);
            logsumset< float > denom_terms(use_multiset_logsum);
            for(size_t j = 0; j < n_components; ++j) {
                float v = log_pdf[i][j] + curr_mixture.log_weights[j];
                log_resp[i][j] = v;
                denom_terms.add(v);
            }
            float log_denom = denom_terms.val();
            for(size_t j = 0; j < n_components; ++j) {
                log_resp[i][j] -= log_denom;
                LOG("training_core", debug)
                    << "resp " << i << " " << j << " "
                    << std::scientific << std::exp(log_resp[i][j]) << endl;
            }
        }

        // update weights
        //
        //   w'[j] := sum_i resp[i][j] / n_data
        //
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                numer_terms.add(log_resp[i][j]);
            }
            float log_numer = numer_terms.val();
            new_mixture.log_weights[j] = log_numer - log_n_data;
        }

        // update means
        //
        //   mu_j := sum_i ( resp[i][j] * level_mean_i ) / sum_i resp[i][j]
        //         = sum_i ( resp[i][j] * level_mean_i ) / ( w'[j] * n_data )
        //
        vector< float > new_log_mean(2);
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                numer_terms.add(log_resp[i][j] + data[i].log_level_mean);
            }
            float log_numer = numer_terms.val();
            new_log_mean[j] = log_numer - (log_n_data + new_mixture.log_weights[j]);
        }

        // update stdvs
        //
        //   var_j := sum_i ( resp[i][j] * ( ( level_mean_i - mu_j ) / read_var_i )^2 ) / sum_i resp[i][j]
        //          = sum_i ( resp[i][j] * ( ( level_mean_i - mu_j ) / read_var_i )^2 ) / ( w'[j] * n_data )
        //
        vector< float > new_log_var(2);
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                float v = std::abs(data[i].level_mean - std::exp(new_log_mean[j]));
                numer_terms.add(log_resp[i][j] + (not std::isnan(v) and v > 0? 2.0 * (std::log(v) - data[i].log_read_var) : 0.0));
            }
            float log_numer = numer_terms.val();
            new_log_var[j] = log_numer - (log_n_data + new_mixture.log_weights[j]);
        }

        for(size_t j = 0; j < n_components; ++j) {
            new_mixture.params[j].level_mean = std::exp(new_log_mean[j]);
            new_mixture.params[j].level_log_stdv = .5 * new_log_var[j];
            new_mixture.params[j].level_stdv = std::exp(new_mixture.params[j].level_log_stdv);
            LOG("training_core", info)
                << "new_mixture " << iteration << " " << j << " "
                << std::scientific << std::exp(new_mixture.log_weights[j]) << " "
                << new_mixture.params[j].level_mean << " " << new_mixture.params[j].level_stdv << endl;
        }

        curr_mixture = new_mixture;
    }
    return curr_mixture;
}

InvGaussianMixture train_invgaussian_mixture(const vector< StateTrainingData >& data, const InvGaussianMixture& in_mixture)
{
    size_t n_components = in_mixture.params.size();
    assert(in_mixture.log_weights.size() == n_components);
    size_t n_data = data.size();
    auto crt_mixture = in_mixture;

    // compute gaussian pdfs
    //
    //   pdf[i][j].first = gauss(mu_j, sigma_j * read_var_i, level_mean_i)
    //
    vector< vector< std::pair< float, float > > > log_pdf(n_data);
    for (size_t i = 0; i < n_data; ++i) {
        log_pdf[i].resize(n_components);
        for (size_t j = 0; j < n_components; ++j) {
            PoreModelStateParams scaled_state = in_mixture.params[j];
            scaled_state.level_stdv *= data[i].read_var;
            scaled_state.level_log_stdv += data[i].log_read_var;
            log_pdf[i][j].first = log_normal_pdf(data[i].level_mean, scaled_state);
            assert(not std::isnan(log_pdf[i][j].first));
            LOG("training_core", debug)
                << "log_gauss_pdf " << i << " " << j << " "
                << std::scientific << std::exp(log_pdf[i][j].first) << endl;
        }
    }

    // compute gaussian weights
    //
    //   g_weights[i][j] := ( w_j * pdf[i][j].first ) / sum_k ( w_k * pdf[i][k].first )
    //
    vector< vector< float > > log_g_weights(n_data);
    for (size_t i = 0; i < n_data; ++i) {
        log_g_weights[i].resize(n_components);
        logsumset< float > denom_terms(use_multiset_logsum);
        for (size_t j = 0; j < n_components; ++j) {
            float v = in_mixture.log_weights[j] + log_pdf[i][j].first;
            log_g_weights[i][j] = v;
            denom_terms.add(v);
        }
        float log_denom = denom_terms.val();
        for (size_t j = 0; j < n_components; ++j) {
            log_g_weights[i][j] -= log_denom;
            LOG("training_core", debug)
                << "g_weights " << i << " " << j << " "
                << std::scientific << std::exp(log_g_weights[i][j]) << endl;
        }
    }

    for (size_t iteration = 0; iteration < 10; ++iteration) {
        // compute inverse gaussian pdfs
        //
        //   pdf[i][j].second = invgauss(eta_j, lambda_j * ( read_var_sd_i / read_var_scale_i ), level_stdv_i)
        //
        for (size_t i = 0; i < n_data; ++i) {
            for (size_t j = 0; j < n_components; ++j) {
                PoreModelStateParams scaled_state = in_mixture.params[j];
                scaled_state.sd_lambda *= data[i].read_var_sd / data[i].read_scale_sd;
                scaled_state.sd_log_lambda += data[i].log_read_var_sd - data[i].log_read_scale_sd;
                log_pdf[i][j].second = log_invgauss_pdf(data[i].level_stdv, data[i].log_level_stdv, scaled_state);
                assert(not std::isnan(log_pdf[i][j].second));
                LOG("training_core", debug)
                    << "invgauss_pdf " << i << " " << j << " "
                    << std::scientific << std::exp(log_pdf[i][j].second) << endl;
            }
        }
        // compute inverse gaussian weights (responsibilities)
        //
        //   ig_weights[i][j] := ( g_weights[i][j] * pdf[i][j].second ) / sum_k ( g_weights[i][k] * pdf[i][k].second )
        //
        vector< vector< float > > log_ig_weights(n_data);
        for (size_t i = 0; i < n_data; ++i) {
            log_ig_weights[i].resize(n_components);
            logsumset< float > denom_terms(use_multiset_logsum);
            for (size_t j = 0; j < n_components; ++j) {
                float v = log_g_weights[i][j] + log_pdf[i][j].second;
                log_ig_weights[i][j] = v;
                denom_terms.add(v);
            }
            float log_denom = denom_terms.val();
            for (size_t j = 0; j < n_components; ++j) {
                log_ig_weights[i][j] -= log_denom;
                LOG("training_core", debug)
                    << "ig_weights " << i << " " << j << " "
                    << std::scientific << std::exp(log_ig_weights[i][j]) << endl;
            }
        }

        // update eta
        //
        //   eta_j := sum_i ( ig_weigts[i][j] * lambda'_ij * level_stdv_i ) / sum_i ( ig_weights[i][j] * lambda'_ij )
        //   lambda'_ij := lambda_j * ( read_var_sd_i / read_var_scale_i )
        //
        auto new_mixture = crt_mixture;
        for (size_t j = 0; j < n_components; ++j) {
            logsumset< float > numer_terms(use_multiset_logsum);
            logsumset< float > denom_terms(use_multiset_logsum);
            for (size_t i = 0; i < n_data; ++i) {
                float v = log_ig_weights[i][j] + in_mixture.params[j].sd_log_lambda + (data[i].log_read_var_sd - data[i].log_read_scale_sd);
                numer_terms.add(v + data[i].log_level_stdv);
                denom_terms.add(v);
            }
            float log_numer = numer_terms.val();
            float log_denom = denom_terms.val();
            new_mixture.params[j].sd_mean = std::exp(log_numer - log_denom);
            LOG("training_core", info)
                << "new_mixture " << iteration << " " << j << " "
                << std::scientific << std::exp(new_mixture.log_weights[j]) << " "
                << new_mixture.params[j].sd_mean << endl;
        }
        std::swap(crt_mixture, new_mixture);
    } // for iteration

    return crt_mixture;
} // train_ig_mixture
<|endoftext|>


<commit_before>// http://www.nuonsoft.com/blog/2017/08/10/implementing-a-thread-safe-singleton-with-c11-using-magic-statics/
// http://blog.mbedded.ninja/programming/languages/c-plus-plus/magic-statics

#include <iostream>

class CSingleton final {
 public:
  static CSingleton& GetInstance();
  int getValue() const { return mValue; }

 private:
  CSingleton() = default;
  ~CSingleton() = default;

  CSingleton(const CSingleton&) = delete;
  CSingleton& operator=(const CSingleton&) = delete;
  CSingleton(CSingleton&&) = delete;
  CSingleton& operator=(CSingleton&&) = delete;

  int mValue = 0;
};

CSingleton& CSingleton::GetInstance() {
  static CSingleton instance;
  return instance;
}

int main() {
  auto const value = CSingleton::GetInstance().getValue();
  std::cout << value << '\n';
}
<commit_msg>Brace initialization.<commit_after>// http://www.nuonsoft.com/blog/2017/08/10/implementing-a-thread-safe-singleton-with-c11-using-magic-statics/
// http://blog.mbedded.ninja/programming/languages/c-plus-plus/magic-statics

#include <iostream>

class CSingleton final {
 public:
  static CSingleton& GetInstance();
  int getValue() const { return mValue; }

 private:
  CSingleton() = default;
  ~CSingleton() = default;

  CSingleton(const CSingleton&) = delete;
  CSingleton& operator=(const CSingleton&) = delete;
  CSingleton(CSingleton&&) = delete;
  CSingleton& operator=(CSingleton&&) = delete;

  int mValue = 0;
};

CSingleton& CSingleton::GetInstance() {
  static CSingleton instance;
  return instance;
}

int main() {
  auto const value{CSingleton::GetInstance().getValue()};
  std::cout << value << '\n';
}
<|endoftext|>


<commit_before>// http://www.nuonsoft.com/blog/2017/08/10/implementing-a-thread-safe-singleton-with-c11-using-magic-statics/
// http://blog.mbedded.ninja/programming/languages/c-plus-plus/magic-statics

#include <iostream>

class CSingleton final {
 public:
  static CSingleton& GetInstance();
  int getValue() const { return mValue; }

 private:
  CSingleton() = default;
  ~CSingleton() = default;

  CSingleton(const CSingleton&) = delete;
  CSingleton& operator=(const CSingleton&) = delete;
  CSingleton(CSingleton&&) = delete;
  CSingleton& operator=(CSingleton&&) = delete;

  int mValue = 0;
};

CSingleton& CSingleton::GetInstance() {
  static CSingleton instance;
  return instance;
}

int main() {
  auto const value{CSingleton::GetInstance().getValue()};
  std::cout << value << '\n';
}
<commit_msg>Deleted member function should be public.<commit_after>// http://www.nuonsoft.com/blog/2017/08/10/implementing-a-thread-safe-singleton-with-c11-using-magic-statics/
// http://blog.mbedded.ninja/programming/languages/c-plus-plus/magic-statics

#include <iostream>

class CSingleton final {
 public:
  static CSingleton& GetInstance();
  int getValue() const { return mValue; }

  CSingleton(const CSingleton&) = delete;
  CSingleton& operator=(const CSingleton&) = delete;
  CSingleton(CSingleton&&) = delete;
  CSingleton& operator=(CSingleton&&) = delete;

 private:
  CSingleton() = default;
  ~CSingleton() = default;

  int mValue = 0;
};

CSingleton& CSingleton::GetInstance() {
  static CSingleton instance;
  return instance;
}

int main() {
  auto const value{CSingleton::GetInstance().getValue()};
  std::cout << value << '\n';
}
<|endoftext|>


<commit_before>/*
Tests the analog output data integrity in three different domains.

FPGA: before serialization on the FPGA
LVDS: arriving at DAC
SYNC: on output clock on DAC
*/

#include <functional>
#include <iostream>
#include <random>

#include "../C++/helpers.h"
#include "concol.h"
#include "libaps2.h"

using std::cout;
using std::endl;
using std::string;
using std::vector;

void print_results(const vector<uint32_t> &results) {

  auto print_pass_fail = [](const bool &result) {
    if (result) {
      cout << concol::GREEN << "pass" << concol::RESET;
    } else {
      cout << concol::RED << "fail" << concol::RESET;
    }
  };

  cout << "FPGA: ";
  print_pass_fail(results[2] == results[0]);
  cout << " / ";
  print_pass_fail(results[3] == results[1]);

  cout << " LVDS: ";
  print_pass_fail(results[4] == results[0]);
  cout << " / ";
  print_pass_fail(results[5] == results[1]);

  cout << " SYNC: ";
  print_pass_fail(results[6] == results[4]);
  cout << " / ";
  print_pass_fail(results[7] == results[5]);
}

int main(int argc, char const *argv[]) {

  print_title("BBN APS2 Analog Data integrity Test");

  // Poll for which device to test
  string deviceSerial = get_device_id();
  if (deviceSerial.empty()) {
    cout << concol::RED << "No APS2 devices connected! Exiting..."
         << concol::RESET << endl;
    return 0;
  }

  set_logging_level(logDEBUG1);

  connect_APS(deviceSerial.c_str());
  stop(deviceSerial.c_str());

  // force initialize device
  init_APS(deviceSerial.c_str(), 1);

  // Generate random bit values
  std::default_random_engine generator;
  std::uniform_int_distribution<int> bitDistribution(0, 1);
  std::uniform_int_distribution<int16_t> wordDistribution(-8192, 8191);
  auto randbit = std::bind(bitDistribution, generator);
  auto randWord = std::bind(wordDistribution, generator);

  vector<int16_t> testVec;
  vector<uint32_t> results(8);

  // Loop through DACs
  for (int dac = 0; dac < 2; ++dac) {
    cout << concol::CYAN << "DAC " << dac << concol::RESET << endl;

    for (int bit = 0; bit < 14; ++bit) {
      cout << "Testing bit " << bit << ": ";
      testVec.clear();
      for (int ct = 0; ct < 1000; ++ct) {
        testVec.push_back((randbit() << bit) * (bit == 13 ? -1 : 1));
      }
      run_DAC_BIST(deviceSerial.c_str(), dac, testVec.data(), testVec.size(),
                   results.data());

      print_results(results);
      cout << endl;
    }
    cout << endl;
    cout << "Testing random waveform... ";
    testVec.clear();
    for (int ct = 0; ct < (1 << 17); ++ct) {
      testVec.push_back(randWord());
    }
    run_DAC_BIST(deviceSerial.c_str(), dac, testVec.data(), testVec.size(),
                 results.data());

    print_results(results);

    cout << endl;
  }

  disconnect_APS(deviceSerial.c_str());

  return 0;
}
<commit_msg>tweak DAC_BIST printing to report LVDS capture passes when it matches FPGA<commit_after>/*
Tests the analog output data integrity in three different domains.

FPGA: before serialization on the FPGA
LVDS: arriving at DAC
SYNC: on output clock on DAC
*/

#include <functional>
#include <iostream>
#include <random>

#include "../C++/helpers.h"
#include "concol.h"
#include "libaps2.h"

using std::cout;
using std::endl;
using std::string;
using std::vector;

void print_results(const vector<uint32_t> &results) {

  auto print_pass_fail = [](const bool &result) {
    if (result) {
      cout << concol::GREEN << "pass" << concol::RESET;
    } else {
      cout << concol::RED << "fail" << concol::RESET;
    }
  };

  cout << "FPGA: ";
  print_pass_fail(results[2] == results[0]);
  cout << " / ";
  print_pass_fail(results[3] == results[1]);

  cout << " LVDS: ";
  print_pass_fail(results[4] == results[2]);
  cout << " / ";
  print_pass_fail(results[5] == results[3]);

  cout << " SYNC: ";
  print_pass_fail(results[6] == results[4]);
  cout << " / ";
  print_pass_fail(results[7] == results[5]);
}

int main(int argc, char const *argv[]) {

  print_title("BBN APS2 Analog Data integrity Test");

  // Poll for which device to test
  string deviceSerial = get_device_id();
  if (deviceSerial.empty()) {
    cout << concol::RED << "No APS2 devices connected! Exiting..."
         << concol::RESET << endl;
    return 0;
  }

  set_logging_level(logDEBUG1);

  connect_APS(deviceSerial.c_str());
  stop(deviceSerial.c_str());

  // force initialize device
  init_APS(deviceSerial.c_str(), 1);

  // Generate random bit values
  std::default_random_engine generator;
  std::uniform_int_distribution<int> bitDistribution(0, 1);
  std::uniform_int_distribution<int16_t> wordDistribution(-8192, 8191);
  auto randbit = std::bind(bitDistribution, generator);
  auto randWord = std::bind(wordDistribution, generator);

  vector<int16_t> testVec;
  vector<uint32_t> results(8);

  // Loop through DACs
  for (int dac = 0; dac < 2; ++dac) {
    cout << concol::CYAN << "DAC " << dac << concol::RESET << endl;

    for (int bit = 0; bit < 14; ++bit) {
      cout << "Testing bit " << bit << ": ";
      testVec.clear();
      for (int ct = 0; ct < 1000; ++ct) {
        testVec.push_back((randbit() << bit) * (bit == 13 ? -1 : 1));
      }
      run_DAC_BIST(deviceSerial.c_str(), dac, testVec.data(), testVec.size(),
                   results.data());

      print_results(results);
      cout << endl;
    }
    cout << endl;
    cout << "Testing random waveform... ";
    testVec.clear();
    for (int ct = 0; ct < (1 << 17); ++ct) {
      testVec.push_back(randWord());
    }
    run_DAC_BIST(deviceSerial.c_str(), dac, testVec.data(), testVec.size(),
                 results.data());

    print_results(results);

    cout << endl;
  }

  disconnect_APS(deviceSerial.c_str());

  return 0;
}
<|endoftext|>