<commit_before>/*
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * As a special exception, you may use this file as part of a free
 * software library without restriction. Specifically, if other files
 * instantiate templates or use macros or inline functions from this
 * file, or you compile this file and link it with other files to
 * produce an executable, this file does not by itself cause the
 * resulting executable to be covered by the GNU General Public
 * License. This exception does not however invalidate any other
 * reasons why the executable file might be covered by the GNU Library
 * General Public License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
#include "clockimpl.h"
#include <sys/time.h>
#include <time.h>

namespace cxxtools
{

ClockImpl::ClockImpl()
{}

ClockImpl::~ClockImpl()
{}

void ClockImpl::start()
{
#ifdef HAVE_CLOCK_GETTIME
    clock_gettime(CLOCK_MONOTONIC, &_startTime);
#else
    gettimeofday( &_startTime, 0 );
#endif
}

Timespan ClockImpl::stop() const
{
#ifdef HAVE_CLOCK_GETTIME
    struct timespec stopTime;

    clock_gettime(CLOCK_MONOTONIC, &stopTime);

    time_t secs = stopTime.tv_sec - _startTime.tv_sec;
    suseconds_t usecs = (stopTime.tv_nsec - _startTime.tv_nsec) / 1000;
#else
    struct timeval stopTime;

    gettimeofday( &stopTime, 0 );

    time_t secs = stopTime.tv_sec - _startTime.tv_sec;
    suseconds_t usecs = stopTime.tv_usec - _startTime.tv_usec;
#endif

    return Timespan(secs, usecs);
}

DateTime ClockImpl::getSystemTime()
{
    struct timeval tod;
    gettimeofday(&tod, NULL);

    Date date(tod.tv_sec / 24 / 60 / 60);
    Time time;
    time.setTotalUSecs(tod.tv_usec);
    return DateTime(date, time);
}

DateTime ClockImpl::getLocalTime()
{
    struct timeval tod;
    gettimeofday(&tod, NULL);

    struct tm tim;
    time_t sec = tod.tv_sec;
    localtime_r(&sec, &tim);

    return DateTime( tim.tm_year + 1900,
                     tim.tm_mon + 1,
                     tim.tm_mday,
                     tim.tm_hour,
                     tim.tm_min,
                     tim.tm_sec,
                     0,
                     tod.tv_usec);
}

Timespan ClockImpl::getSystemTicks()
{
    struct timeval tv;
    gettimeofday( &tv, 0 );

    return Timespan(tv.tv_sec, tv.tv_usec);
}

} // namespace System

<commit_msg>fix cxxtools::Clock::getSystemTime<commit_after>/*
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * As a special exception, you may use this file as part of a free
 * software library without restriction. Specifically, if other files
 * instantiate templates or use macros or inline functions from this
 * file, or you compile this file and link it with other files to
 * produce an executable, this file does not by itself cause the
 * resulting executable to be covered by the GNU General Public
 * License. This exception does not however invalidate any other
 * reasons why the executable file might be covered by the GNU Library
 * General Public License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
#include "clockimpl.h"
#include <sys/time.h>
#include <time.h>

namespace cxxtools
{

ClockImpl::ClockImpl()
{}

ClockImpl::~ClockImpl()
{}

void ClockImpl::start()
{
#ifdef HAVE_CLOCK_GETTIME
    clock_gettime(CLOCK_MONOTONIC, &_startTime);
#else
    gettimeofday( &_startTime, 0 );
#endif
}

Timespan ClockImpl::stop() const
{
#ifdef HAVE_CLOCK_GETTIME
    struct timespec stopTime;

    clock_gettime(CLOCK_MONOTONIC, &stopTime);

    time_t secs = stopTime.tv_sec - _startTime.tv_sec;
    suseconds_t usecs = (stopTime.tv_nsec - _startTime.tv_nsec) / 1000;
#else
    struct timeval stopTime;

    gettimeofday( &stopTime, 0 );

    time_t secs = stopTime.tv_sec - _startTime.tv_sec;
    suseconds_t usecs = stopTime.tv_usec - _startTime.tv_usec;
#endif

    return Timespan(secs, usecs);
}

DateTime ClockImpl::getSystemTime()
{
    struct timeval tod;
    gettimeofday(&tod, NULL);

    struct tm tim;
    time_t sec = tod.tv_sec;
    gmtime_r(&sec, &tim);

    return DateTime( tim.tm_year + 1900,
                     tim.tm_mon + 1,
                     tim.tm_mday,
                     tim.tm_hour,
                     tim.tm_min,
                     tim.tm_sec,
                     0,
                     tod.tv_usec);
}

DateTime ClockImpl::getLocalTime()
{
    struct timeval tod;
    gettimeofday(&tod, NULL);

    struct tm tim;
    time_t sec = tod.tv_sec;
    localtime_r(&sec, &tim);

    return DateTime( tim.tm_year + 1900,
                     tim.tm_mon + 1,
                     tim.tm_mday,
                     tim.tm_hour,
                     tim.tm_min,
                     tim.tm_sec,
                     0,
                     tod.tv_usec);
}

Timespan ClockImpl::getSystemTicks()
{
    struct timeval tv;
    gettimeofday( &tv, 0 );

    return Timespan(tv.tv_sec, tv.tv_usec);
}

} // namespace System

<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "FeatureIndex.h"
#include "DocStore.h"
#include "IndexChangeRequest.h"
#include "FeatureIndexWriter.h"
#include "ItemRef.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;
fnord::thread::EventLoop ev;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

   flags.defineFlag(
      "nocache",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no cache",
      "<bool>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  auto enable_cache = !flags.isSet("nocache");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8\n" \
      "    cache=$9",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval,
      enable_cache);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      flags.getString("datadir"),
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* open index */
  auto index_path = FileUtil::joinPaths(flags.getString("index"), "db");
  RefPtr<FeatureIndexWriter> index(new FeatureIndexWriter(index_path, true));

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(
      joined_sessions_schema,
      &analyzer,
      index,
      dry_run);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("feedserver_addr"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, enable_cache, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  if (!dry_run) {
    logjoin_upload.upload();
  }

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cache_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    if (dry_run) {
      txn->abort();
    } else {
      txn->commit();

      try {
        logjoin_upload.upload();
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "upload failed");
      }
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  //exit(0); // FIXPAUL

  return 0;
}

<commit_msg>flags<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "FeatureIndex.h"
#include "DocStore.h"
#include "IndexChangeRequest.h"
#include "FeatureIndexWriter.h"
#include "ItemRef.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;
fnord::thread::EventLoop ev;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "publish_to",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

   flags.defineFlag(
      "nocache",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no cache",
      "<bool>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  auto enable_cache = !flags.isSet("nocache");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8\n" \
      "    cache=$9",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval,
      enable_cache);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      flags.getString("datadir"),
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* open index */
  auto index_path = FileUtil::joinPaths(flags.getString("index"), "db");
  RefPtr<FeatureIndexWriter> index(new FeatureIndexWriter(index_path, true));

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(
      joined_sessions_schema,
      &analyzer,
      index,
      dry_run);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("publish_to"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, enable_cache, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  if (!dry_run) {
    logjoin_upload.upload();
  }

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cache_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    if (dry_run) {
      txn->abort();
    } else {
      txn->commit();

      try {
        logjoin_upload.upload();
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "upload failed");
      }
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  //exit(0); // FIXPAUL

  return 0;
}

<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "cmdata",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "clickmatcher app data dir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  //fnord::iputs("$0", joined_sessions_schema.toString());

  /* set up cmdata */
  auto cmdata_path = flags.getString("cmdata");
  if (!FileUtil::isDirectory(cmdata_path)) {
    RAISEF(kIOError, "no such directory: $0", cmdata_path);
  }

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  //input_feeds.emplace(
  //    "tracker_log.feedserver01.nue01.production.fnrd.net",
  //    URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(joined_sessions_schema, &analyzer);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("feedserver_addr"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  logjoin_upload.upload();

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_iter;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();
    logjoin.flush(txn.get(), watermarks.first);

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5$6",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        stream_offsets_str);

    txn->commit();

    try {
      logjoin_upload.upload();
    } catch (const std::exception& e) {
      fnord::logError("cm.logjoin", e, "upload failed");
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  ev.shutdown();
  //evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  exit(0); // FIXPAUL

  return 0;
}

<commit_msg>cleaning up<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "cmdata",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "clickmatcher app data dir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* set up cmdata */
  auto cmdata_path = flags.getString("cmdata");
  if (!FileUtil::isDirectory(cmdata_path)) {
    RAISEF(kIOError, "no such directory: $0", cmdata_path);
  }

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(joined_sessions_schema, &analyzer);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("feedserver_addr"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  logjoin_upload.upload();

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_iter;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();
    logjoin.flush(txn.get(), watermarks.first);

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5$6",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        stream_offsets_str);

    txn->commit();

    try {
      logjoin_upload.upload();
    } catch (const std::exception& e) {
      fnord::logError("cm.logjoin", e, "upload failed");
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  ev.shutdown();
  //evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  exit(0); // FIXPAUL

  return 0;
}

<|endoftext|>


<commit_before>/*
   Copyright (c) 2009-2011, Jack Poulson
   All rights reserved.

   This file is part of Elemental.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

    - Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.

    - Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

    - Neither the name of the owner nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
*/
#include "elemental/environment.hpp"
using namespace std;
using namespace elemental;
using namespace elemental::imports;

elemental::Grid::Grid
( mpi::Comm comm )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif
    _inGrid = true; // this is true by assumption for this constructor

    // Extract our rank, the underlying group, and the number of processes
    mpi::CommDup( comm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );
    _p = mpi::CommSize( _viewingComm );

    // All processes own the grid, so we have to trivially split _viewingGroup
    _owningGroup = _viewingGroup;
    _notOwningGroup = mpi::GROUP_EMPTY;
    _owningRank = _viewingRank;

    // Factor p
    _r = static_cast<int>(sqrt(static_cast<double>(_p)));
    while( _p % _r != 0 )
        ++_r;
    _c = _p / _r;

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::Grid
( mpi::Comm comm, int r, int c )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif
    _inGrid = true; // this is true by assumption for this constructor

    // Extract our rank, the underlying group, and the number of processes
    mpi::CommDup( comm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );
    _p = mpi::CommSize( _viewingComm );

    // All processes own the grid, so we have to trivially split _viewingGroup
    _owningGroup = _viewingGroup;
    _notOwningGroup = mpi::GROUP_EMPTY;
    _owningRank = _viewingRank;

    _r = r;
    _c = c;
    if( _r < 0 || _c < 0 )
        throw logic_error( "Process grid dimensions must be non-negative." );

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::Grid
( mpi::Comm viewingComm, mpi::Group owningGroup )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif

    // Extract our rank and the underlying group from the viewing comm
    mpi::CommDup( viewingComm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );

    // Extract our rank and the number of processes from the owning group
    _owningGroup = owningGroup;
    _p = mpi::GroupSize( _owningGroup );
    _owningRank = mpi::GroupRank( _owningGroup );
    _inGrid = ( _owningRank != mpi::UNDEFINED );

    // Create the complement of the owning group
    mpi::GroupDifference( _viewingGroup, _owningGroup, _notOwningGroup );

    // Factor p
    _r = static_cast<int>(sqrt(static_cast<double>(_p)));
    while( _p % _r != 0 )
        ++_r;
    _c = _p / _r;

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

// Currently forces a columnMajor absolute rank on the grid
elemental::Grid::Grid
( mpi::Comm viewingComm, mpi::Group owningGroup, int r, int c )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif

    // Extract our rank and the underlying group from the viewing comm
    mpi::CommDup( viewingComm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );

    // Extract our rank and the number of processes from the owning group
    _owningGroup = owningGroup;
    _p = mpi::GroupSize( _owningGroup );
    _owningRank = mpi::GroupRank( _owningGroup );
    _inGrid = ( _owningRank != mpi::UNDEFINED );

    // Create the complement of the owning group
    mpi::GroupDifference( _viewingGroup, _owningGroup, _notOwningGroup );

    _r = r;
    _c = c;

    if( _r < 0 || _c < 0 )
        throw logic_error( "Process grid dimensions must be non-negative." );

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

void
elemental::Grid::SetUpGrid()
{
#ifndef RELEASE
    PushCallStack("Grid::SetUpGrid");
#endif
    if( _p != _r*_c )
    {
        ostringstream msg;
        msg << "Number of processes must match grid size:\n"
            << "  p=" << _p << ", (r,c)=(" << _r << "," << _c << ")";
        throw logic_error( msg.str() );
    }

    _gcd = utilities::GCD( _r, _c );
    int lcm = _p / _gcd;

#ifndef RELEASE
    if( _owningRank == 0 )
    {
        cout << "Building process grid with:\n"
             << "  p=" << _p << ", (r,c)=(" << _r << "," << _c << ")\n"
             << "  gcd=" << _gcd << endl;
    }
#endif

    // Split the viewing comm into the owning and not owning subsets
    if( _inGrid )
        mpi::CommCreate( _viewingComm, _owningGroup, _owningComm );
    else
        mpi::CommCreate( _viewingComm, _notOwningGroup, _notOwningComm );

    if( _inGrid )
    {
        // Create a cartesian communicator
        int dimensions[2] = { _c, _r };
        int periods[2] = { true, true };
        int reorder = false;
        mpi::CartCreate
        ( _owningComm, 2, dimensions, periods, reorder, _cartComm );

        // Set up the MatrixCol and MatrixRow communicators
        int remainingDimensions[2];
        remainingDimensions[0] = false;
        remainingDimensions[1] = true;
        mpi::CartSub( _cartComm, remainingDimensions, _matrixColComm );
        remainingDimensions[0] = true;
        remainingDimensions[1] = false;
        mpi::CartSub( _cartComm, remainingDimensions, _matrixRowComm );
        _matrixColRank = mpi::CommRank( _matrixColComm );
        _matrixRowRank = mpi::CommRank( _matrixRowComm );

        // Set up the VectorCol and VectorRow communicators
        _vectorColRank = _matrixColRank + _r*_matrixRowRank;
        _vectorRowRank = _matrixRowRank + _c*_matrixColRank;
        mpi::CommSplit( _cartComm, 0, _vectorColRank, _vectorColComm );
        mpi::CommSplit( _cartComm, 0, _vectorRowRank, _vectorRowComm );

        // Compute which diagonal 'path' we're in, and what our rank is, then
        // perform AllGather world to store everyone's info
        _diagPathsAndRanks.resize(2*_p);
        vector<int> myDiagPathAndRank(2);
        myDiagPathAndRank[0] = (_matrixRowRank+_r-_matrixColRank) % _gcd;
        int diagPathRank = 0;
        int row = 0;
        int col = myDiagPathAndRank[0];
        for( int j=0; j<lcm; ++j )
        {
            if( row == _matrixColRank && col == _matrixRowRank )
            {
                myDiagPathAndRank[1] = diagPathRank;
                break;
            }
            else
            {
                row = (row + 1) % _r;
                col = (col + 1) % _c;
                ++diagPathRank;
            }
        }
        mpi::AllGather
        ( &myDiagPathAndRank[0], 2, &_diagPathsAndRanks[0], 2, _vectorColComm );

#ifndef RELEASE
        mpi::ErrorHandlerSet( _matrixColComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _matrixRowComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _vectorColComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _vectorRowComm, mpi::ERRORS_RETURN );
#endif
    }
    else
    {
        // diag paths and ranks are implicitly set to undefined
        _matrixColRank = mpi::UNDEFINED;
        _matrixRowRank = mpi::UNDEFINED;
        _vectorColRank = mpi::UNDEFINED;
        _vectorRowRank = mpi::UNDEFINED;
    }

    // Set up the map from the VC group to the _viewingGroup ranks.
    // Since the VC communicator preserves the ordering of the _owningGroup
    // ranks, we can simply translate from _owningGroup.
    std::vector<int> ranks(_p);
    for( int i=0; i<_p; ++i )
        ranks[i] = i;
    _VCToViewingMap.resize(_p);
    mpi::GroupTranslateRanks
    ( _owningGroup, _p, &ranks[0], _viewingGroup, &_VCToViewingMap[0] );

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::~Grid()
{
    if( !mpi::Finalized() )
    {
        if( _inGrid )
        {
            mpi::CommFree( _matrixColComm );
            mpi::CommFree( _matrixRowComm );
            mpi::CommFree( _vectorColComm );
            mpi::CommFree( _vectorRowComm );
            mpi::CommFree( _cartComm );
        }

        if( _inGrid )
            mpi::CommFree( _owningComm );
        else
            mpi::CommFree( _notOwningComm );

        if( _notOwningGroup != mpi::GROUP_EMPTY )
            mpi::GroupFree( _notOwningGroup );

        mpi::CommFree( _viewingComm );
    }
}

<commit_msg>Removed illegal but functional use of MPI_Comm_create in favor of MPI_Comm_split.<commit_after>/*
   Copyright (c) 2009-2011, Jack Poulson
   All rights reserved.

   This file is part of Elemental.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

    - Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.

    - Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

    - Neither the name of the owner nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
*/
#include "elemental/environment.hpp"
using namespace std;
using namespace elemental;
using namespace elemental::imports;

elemental::Grid::Grid
( mpi::Comm comm )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif
    _inGrid = true; // this is true by assumption for this constructor

    // Extract our rank, the underlying group, and the number of processes
    mpi::CommDup( comm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );
    _p = mpi::CommSize( _viewingComm );

    // All processes own the grid, so we have to trivially split _viewingGroup
    _owningGroup = _viewingGroup;
    _notOwningGroup = mpi::GROUP_EMPTY;
    _owningRank = _viewingRank;

    // Factor p
    _r = static_cast<int>(sqrt(static_cast<double>(_p)));
    while( _p % _r != 0 )
        ++_r;
    _c = _p / _r;

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::Grid
( mpi::Comm comm, int r, int c )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif
    _inGrid = true; // this is true by assumption for this constructor

    // Extract our rank, the underlying group, and the number of processes
    mpi::CommDup( comm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );
    _p = mpi::CommSize( _viewingComm );

    // All processes own the grid, so we have to trivially split _viewingGroup
    _owningGroup = _viewingGroup;
    _notOwningGroup = mpi::GROUP_EMPTY;
    _owningRank = _viewingRank;

    _r = r;
    _c = c;
    if( _r < 0 || _c < 0 )
        throw logic_error( "Process grid dimensions must be non-negative." );

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::Grid
( mpi::Comm viewingComm, mpi::Group owningGroup )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif

    // Extract our rank and the underlying group from the viewing comm
    mpi::CommDup( viewingComm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );

    // Extract our rank and the number of processes from the owning group
    _owningGroup = owningGroup;
    _p = mpi::GroupSize( _owningGroup );
    _owningRank = mpi::GroupRank( _owningGroup );
    _inGrid = ( _owningRank != mpi::UNDEFINED );

    // Create the complement of the owning group
    mpi::GroupDifference( _viewingGroup, _owningGroup, _notOwningGroup );

    // Factor p
    _r = static_cast<int>(sqrt(static_cast<double>(_p)));
    while( _p % _r != 0 )
        ++_r;
    _c = _p / _r;

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

// Currently forces a columnMajor absolute rank on the grid
elemental::Grid::Grid
( mpi::Comm viewingComm, mpi::Group owningGroup, int r, int c )
{
#ifndef RELEASE
    PushCallStack("Grid::Grid");
#endif

    // Extract our rank and the underlying group from the viewing comm
    mpi::CommDup( viewingComm, _viewingComm );
    mpi::CommGroup( _viewingComm, _viewingGroup );
    _viewingRank = mpi::CommRank( _viewingComm );

    // Extract our rank and the number of processes from the owning group
    _owningGroup = owningGroup;
    _p = mpi::GroupSize( _owningGroup );
    _owningRank = mpi::GroupRank( _owningGroup );
    _inGrid = ( _owningRank != mpi::UNDEFINED );

    // Create the complement of the owning group
    mpi::GroupDifference( _viewingGroup, _owningGroup, _notOwningGroup );

    _r = r;
    _c = c;

    if( _r < 0 || _c < 0 )
        throw logic_error( "Process grid dimensions must be non-negative." );

    SetUpGrid();

#ifndef RELEASE
    PopCallStack();
#endif
}

void
elemental::Grid::SetUpGrid()
{
#ifndef RELEASE
    PushCallStack("Grid::SetUpGrid");
#endif
    if( _p != _r*_c )
    {
        ostringstream msg;
        msg << "Number of processes must match grid size:\n"
            << "  p=" << _p << ", (r,c)=(" << _r << "," << _c << ")";
        throw logic_error( msg.str() );
    }

    _gcd = utilities::GCD( _r, _c );
    int lcm = _p / _gcd;

#ifndef RELEASE
    if( _owningRank == 0 )
    {
        cout << "Building process grid with:\n"
             << "  p=" << _p << ", (r,c)=(" << _r << "," << _c << ")\n"
             << "  gcd=" << _gcd << endl;
    }
#endif

    // Split the viewing comm into the owning and not owning subsets
    if( _inGrid )
        mpi::CommSplit( _viewingComm, true, _owningRank, _owningComm );
    else
        mpi::CommSplit( _viewingComm, false, 0, _notOwningComm );

    if( _inGrid )
    {
        // Create a cartesian communicator
        int dimensions[2] = { _c, _r };
        int periods[2] = { true, true };
        int reorder = false;
        mpi::CartCreate
        ( _owningComm, 2, dimensions, periods, reorder, _cartComm );

        // Set up the MatrixCol and MatrixRow communicators
        int remainingDimensions[2];
        remainingDimensions[0] = false;
        remainingDimensions[1] = true;
        mpi::CartSub( _cartComm, remainingDimensions, _matrixColComm );
        remainingDimensions[0] = true;
        remainingDimensions[1] = false;
        mpi::CartSub( _cartComm, remainingDimensions, _matrixRowComm );
        _matrixColRank = mpi::CommRank( _matrixColComm );
        _matrixRowRank = mpi::CommRank( _matrixRowComm );

        // Set up the VectorCol and VectorRow communicators
        _vectorColRank = _matrixColRank + _r*_matrixRowRank;
        _vectorRowRank = _matrixRowRank + _c*_matrixColRank;
        mpi::CommSplit( _cartComm, 0, _vectorColRank, _vectorColComm );
        mpi::CommSplit( _cartComm, 0, _vectorRowRank, _vectorRowComm );

        // Compute which diagonal 'path' we're in, and what our rank is, then
        // perform AllGather world to store everyone's info
        _diagPathsAndRanks.resize(2*_p);
        vector<int> myDiagPathAndRank(2);
        myDiagPathAndRank[0] = (_matrixRowRank+_r-_matrixColRank) % _gcd;
        int diagPathRank = 0;
        int row = 0;
        int col = myDiagPathAndRank[0];
        for( int j=0; j<lcm; ++j )
        {
            if( row == _matrixColRank && col == _matrixRowRank )
            {
                myDiagPathAndRank[1] = diagPathRank;
                break;
            }
            else
            {
                row = (row + 1) % _r;
                col = (col + 1) % _c;
                ++diagPathRank;
            }
        }
        mpi::AllGather
        ( &myDiagPathAndRank[0], 2, &_diagPathsAndRanks[0], 2, _vectorColComm );

#ifndef RELEASE
        mpi::ErrorHandlerSet( _matrixColComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _matrixRowComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _vectorColComm, mpi::ERRORS_RETURN );
        mpi::ErrorHandlerSet( _vectorRowComm, mpi::ERRORS_RETURN );
#endif
    }
    else
    {
        // diag paths and ranks are implicitly set to undefined
        _matrixColRank = mpi::UNDEFINED;
        _matrixRowRank = mpi::UNDEFINED;
        _vectorColRank = mpi::UNDEFINED;
        _vectorRowRank = mpi::UNDEFINED;
    }

    // Set up the map from the VC group to the _viewingGroup ranks.
    // Since the VC communicator preserves the ordering of the _owningGroup
    // ranks, we can simply translate from _owningGroup.
    std::vector<int> ranks(_p);
    for( int i=0; i<_p; ++i )
        ranks[i] = i;
    _VCToViewingMap.resize(_p);
    mpi::GroupTranslateRanks
    ( _owningGroup, _p, &ranks[0], _viewingGroup, &_VCToViewingMap[0] );

#ifndef RELEASE
    PopCallStack();
#endif
}

elemental::Grid::~Grid()
{
    if( !mpi::Finalized() )
    {
        if( _inGrid )
        {
            mpi::CommFree( _matrixColComm );
            mpi::CommFree( _matrixRowComm );
            mpi::CommFree( _vectorColComm );
            mpi::CommFree( _vectorRowComm );
            mpi::CommFree( _cartComm );
        }

        if( _inGrid )
            mpi::CommFree( _owningComm );
        else
            mpi::CommFree( _notOwningComm );

        if( _notOwningGroup != mpi::GROUP_EMPTY )
            mpi::GroupFree( _notOwningGroup );

        mpi::CommFree( _viewingComm );
    }
}

<|endoftext|>


<commit_before>#include "dbmanager.h"
#include <QtSql/QSqlQuery>
#include <QTimeZone>
#include <QDateTime>
#include <QDebug>

DBManager::DBManager(const QString& path, bool doCreate, QObject *parent) : QObject(parent)
{
    qRegisterMetaType<QList<NoteData*> >("QList<NoteData*>");

    m_db = QSqlDatabase::addDatabase("QSQLITE");
    m_db.setDatabaseName(path);

    if (!m_db.open()){
        qDebug() << "Error: connection with database fail";
    }else{
        qDebug() << "Database: connection ok";
    }

    if(doCreate){
        QSqlQuery query;
        QString active = "CREATE TABLE active_notes ("
                         "id INTEGER PRIMARY KEY AUTOINCREMENT,"
                         "creation_date INTEGER NOT NULL DEFAULT (0),"
                         "modification_date INTEGER NOT NULL DEFAULT (0),"
                         "deletion_date INTEGER NOT NULL DEFAULT (0),"
                         "content TEXT, "
                         "full_title TEXT);";

        query.exec(active);

        QString active_index = "CREATE UNIQUE INDEX active_index on active_notes (id ASC);";
        query.exec(active_index);

        QString deleted = "CREATE TABLE deleted_notes ("
                          "id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"
                          "creation_date INTEGER NOT NULL DEFAULT (0),"
                          "modification_date INTEGER NOT NULL DEFAULT (0),"
                          "deletion_date INTEGER NOT NULL DEFAULT (0),"
                          "content TEXT,"
                          "full_title TEXT)";
        query.exec(deleted);
    }

}

bool DBManager::isNoteExist(NoteData* note)
{
    QSqlQuery query;

    int id = note->id().split('_')[1].toInt();
    QString queryStr = QStringLiteral("SELECT EXISTS(SELECT 1 FROM active_notes WHERE id = %1 LIMIT 1 )")
                       .arg(id);
    query.exec(queryStr);
    query.next();

    return query.value(0).toInt() == 1;
}

QList<NoteData *> DBManager::getAllNotes()
{
    QList<NoteData *> noteList;

    QSqlQuery query;
    query.prepare("SELECT * FROM active_notes");
    bool status = query.exec();
    if(status){
        while(query.next()){
            NoteData* note = new NoteData(this);
            int id =  query.value(0).toInt();
            qint64 epochDateTimeCreation = query.value(1).toLongLong();
            QDateTime dateTimeCreation = QDateTime::fromMSecsSinceEpoch(epochDateTimeCreation, QTimeZone::systemTimeZone());
            qint64 epochDateTimeModification= query.value(2).toLongLong();
            QDateTime dateTimeModification = QDateTime::fromMSecsSinceEpoch(epochDateTimeModification, QTimeZone::systemTimeZone());
            QString content = query.value(4).toString();
            QString fullTitle = query.value(5).toString();

            note->setId(QStringLiteral("noteID_%1").arg(id));
            note->setCreationDateTime(dateTimeCreation);
            note->setLastModificationDateTime(dateTimeModification);
            note->setContent(content);
            note->setFullTitle(fullTitle);

            noteList.push_back(note);
        }

        emit notesReceived(noteList);
    }

    return noteList;
}

bool DBManager::addNote(NoteData* note)
{
    QSqlQuery query;

    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    QString content = note->content().replace("'","''");
    QString fullTitle = note->fullTitle().replace("'","''");

    QString queryStr = QString("INSERT INTO active_notes (creation_date, modification_date, deletion_date, content, full_title) "
                               "VALUES (%1, %1, -1, '%2', '%3');")
                       .arg(epochTimeDateCreated)
                       .arg(content)
                       .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);

}

bool DBManager::removeNote(NoteData* note)
{
    QSqlQuery query;

    int id = note->id().split('_')[1].toInt();

    QString queryStr = QStringLiteral("DELETE FROM active_notes "
                                      "WHERE id=%1")
                       .arg(id);
    query.exec(queryStr);
    bool removed = (query.numRowsAffected() == 1);

    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateDeleted = note->deletionDateTime().toMSecsSinceEpoch();
    QString content = note->content().replace("'","''");
    QString fullTitle = note->fullTitle().replace("'","''");

    queryStr = QString("INSERT INTO deleted_notes "
                       "VALUES (%1, %2, %3, %4, '%5', '%6');")
               .arg(id)
               .arg(epochTimeDateCreated)
               .arg(epochTimeDateModified)
               .arg(epochTimeDateDeleted)
               .arg(content)
               .arg(fullTitle);

    query.exec(queryStr);
    bool addedToTrashDB = (query.numRowsAffected() == 1);

    return (removed && addedToTrashDB);
}

bool DBManager::modifyNote(NoteData* note)
{
    QSqlQuery query;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    QString content = note->content().replace("'","''");
    QString fullTitle = note->fullTitle().replace("'","''");

    QString queryStr = QStringLiteral("UPDATE active_notes "
                                      "SET modification_date=%1, content='%2', full_title='%3' "
                                      "WHERE id=%4")
                       .arg(epochTimeDateModified)
                       .arg(content)
                       .arg(fullTitle)
                       .arg(id);
    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

bool DBManager::migrateNote(NoteData* note)
{
    QSqlQuery query;

    QString emptyStr;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QString("INSERT INTO active_notes "
                               "VALUES (%1, %2, %3, -1, '%4', '%5');")
                       .arg(id)
                       .arg(epochTimeDateCreated)
                       .arg(epochTimeDateModified)
                       .arg(content)
                       .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

bool DBManager::migrateTrash(NoteData* note)
{
    QSqlQuery query;

    QString emptyStr;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateDeleted = note->deletionDateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QString("INSERT INTO deleted_notes "
                       "VALUES (%1, %2, %3, %4, '%5', '%6');")
               .arg(id)
               .arg(epochTimeDateCreated)
               .arg(epochTimeDateModified)
               .arg(epochTimeDateDeleted)
               .arg(content)
               .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

int DBManager::getLastRowID()
{
    QSqlQuery query;
    query.exec("SELECT seq from SQLITE_SEQUENCE WHERE name='active_notes';");
    query.next();
    return query.value(0).toInt();
}
<commit_msg>Fix: saving modified note, having \x0 at the end, to database<commit_after>#include "dbmanager.h"
#include <QtSql/QSqlQuery>
#include <QTimeZone>
#include <QDateTime>
#include <QDebug>

DBManager::DBManager(const QString& path, bool doCreate, QObject *parent) : QObject(parent)
{
    qRegisterMetaType<QList<NoteData*> >("QList<NoteData*>");

    m_db = QSqlDatabase::addDatabase("QSQLITE");
    m_db.setDatabaseName(path);

    if (!m_db.open()){
        qDebug() << "Error: connection with database fail";
    }else{
        qDebug() << "Database: connection ok";
    }

    if(doCreate){
        QSqlQuery query;
        QString active = "CREATE TABLE active_notes ("
                         "id INTEGER PRIMARY KEY AUTOINCREMENT,"
                         "creation_date INTEGER NOT NULL DEFAULT (0),"
                         "modification_date INTEGER NOT NULL DEFAULT (0),"
                         "deletion_date INTEGER NOT NULL DEFAULT (0),"
                         "content TEXT, "
                         "full_title TEXT);";

        query.exec(active);

        QString active_index = "CREATE UNIQUE INDEX active_index on active_notes (id ASC);";
        query.exec(active_index);

        QString deleted = "CREATE TABLE deleted_notes ("
                          "id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"
                          "creation_date INTEGER NOT NULL DEFAULT (0),"
                          "modification_date INTEGER NOT NULL DEFAULT (0),"
                          "deletion_date INTEGER NOT NULL DEFAULT (0),"
                          "content TEXT,"
                          "full_title TEXT)";
        query.exec(deleted);
    }

}

bool DBManager::isNoteExist(NoteData* note)
{
    QSqlQuery query;

    int id = note->id().split('_')[1].toInt();
    QString queryStr = QStringLiteral("SELECT EXISTS(SELECT 1 FROM active_notes WHERE id = %1 LIMIT 1 )")
                       .arg(id);
    query.exec(queryStr);
    query.next();

    return query.value(0).toInt() == 1;
}

QList<NoteData *> DBManager::getAllNotes()
{
    QList<NoteData *> noteList;

    QSqlQuery query;
    query.prepare("SELECT * FROM active_notes");
    bool status = query.exec();
    if(status){
        while(query.next()){
            NoteData* note = new NoteData(this);
            int id =  query.value(0).toInt();
            qint64 epochDateTimeCreation = query.value(1).toLongLong();
            QDateTime dateTimeCreation = QDateTime::fromMSecsSinceEpoch(epochDateTimeCreation, QTimeZone::systemTimeZone());
            qint64 epochDateTimeModification= query.value(2).toLongLong();
            QDateTime dateTimeModification = QDateTime::fromMSecsSinceEpoch(epochDateTimeModification, QTimeZone::systemTimeZone());
            QString content = query.value(4).toString();
            QString fullTitle = query.value(5).toString();

            note->setId(QStringLiteral("noteID_%1").arg(id));
            note->setCreationDateTime(dateTimeCreation);
            note->setLastModificationDateTime(dateTimeModification);
            note->setContent(content);
            note->setFullTitle(fullTitle);

            noteList.push_back(note);
        }

        emit notesReceived(noteList);
    }

    return noteList;
}

bool DBManager::addNote(NoteData* note)
{
    QSqlQuery query;
    QString emptyStr;

    qint64 epochTimeDateCreated = note->creationDateTime()
                                  .toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QString("INSERT INTO active_notes (creation_date, modification_date, deletion_date, content, full_title) "
                               "VALUES (%1, %1, -1, '%2', '%3');")
                       .arg(epochTimeDateCreated)
                       .arg(content)
                       .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);

}

bool DBManager::removeNote(NoteData* note)
{
    QSqlQuery query;
    QString emptyStr;

    int id = note->id().split('_')[1].toInt();

    QString queryStr = QStringLiteral("DELETE FROM active_notes "
                                      "WHERE id=%1")
                       .arg(id);
    query.exec(queryStr);
    bool removed = (query.numRowsAffected() == 1);

    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateDeleted = note->deletionDateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    queryStr = QString("INSERT INTO deleted_notes "
                       "VALUES (%1, %2, %3, %4, '%5', '%6');")
               .arg(id)
               .arg(epochTimeDateCreated)
               .arg(epochTimeDateModified)
               .arg(epochTimeDateDeleted)
               .arg(content)
               .arg(fullTitle);

    query.exec(queryStr);
    bool addedToTrashDB = (query.numRowsAffected() == 1);

    return (removed && addedToTrashDB);
}

bool DBManager::modifyNote(NoteData* note)
{
    QSqlQuery query;
    QString emptyStr;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QStringLiteral("UPDATE active_notes "
                                      "SET modification_date=%1, content='%2', full_title='%3' "
                                      "WHERE id=%4")
                       .arg(epochTimeDateModified)
                       .arg(content)
                       .arg(fullTitle)
                       .arg(id);
    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

bool DBManager::migrateNote(NoteData* note)
{
    QSqlQuery query;

    QString emptyStr;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QString("INSERT INTO active_notes "
                               "VALUES (%1, %2, %3, -1, '%4', '%5');")
                       .arg(id)
                       .arg(epochTimeDateCreated)
                       .arg(epochTimeDateModified)
                       .arg(content)
                       .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

bool DBManager::migrateTrash(NoteData* note)
{
    QSqlQuery query;
    QString emptyStr;

    int id = note->id().split('_')[1].toInt();
    qint64 epochTimeDateCreated = note->creationDateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateModified = note->lastModificationdateTime().toMSecsSinceEpoch();
    qint64 epochTimeDateDeleted = note->deletionDateTime().toMSecsSinceEpoch();
    QString content = note->content()
                      .replace("'","''")
                      .replace(QChar('\x0'), emptyStr);
    QString fullTitle = note->fullTitle()
                        .replace("'","''")
                        .replace(QChar('\x0'), emptyStr);

    QString queryStr = QString("INSERT INTO deleted_notes "
                       "VALUES (%1, %2, %3, %4, '%5', '%6');")
               .arg(id)
               .arg(epochTimeDateCreated)
               .arg(epochTimeDateModified)
               .arg(epochTimeDateDeleted)
               .arg(content)
               .arg(fullTitle);

    query.exec(queryStr);
    return (query.numRowsAffected() == 1);
}

int DBManager::getLastRowID()
{
    QSqlQuery query;
    query.exec("SELECT seq from SQLITE_SEQUENCE WHERE name='active_notes';");
    query.next();
    return query.value(0).toInt();
}
<|endoftext|>