<commit_before>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 Paul Asmuth, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stdlib.h>
#include <unistd.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-mdb/MDB.h"
#include "cm-common/CustomerNamespace.h"
#include "cm-logjoin/LogJoin.h"

using namespace fnord;

struct FeedChunkInfo {
  uint64_t generation;
  uint64_t generation_window;
  uint64_t start_time;
  uint64_t end_time;
  HashMap<String, String> start_offsets;

  template <typename T>
  static void reflect(T* meta) {
    meta->prop(&FeedChunkInfo::generation, 1, "generation", false);
    meta->prop(&FeedChunkInfo::generation_window, 2, "generation_window", false);
    meta->prop(&FeedChunkInfo::start_time, 3, "start_time", false);
    meta->prop(&FeedChunkInfo::end_time, 4, "end_time", false);
    meta->prop(&FeedChunkInfo::start_offsets, 5, "start_offsets", false);
  };
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_path",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "output_prefix",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output filename prefix",
      "<prefix>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "generation_window_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "14400",
      "generation_window_secs",
      "<secs>");

  flags.defineFlag(
      "generation_delay_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "3600",
      "generation_delay_secs",
      "<secs>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  auto output_path = flags.getString("output_path");
  auto output_prefix = flags.getString("output_prefix");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t generation_window_micros =
      flags.getInt("generation_window_secs") * kMicrosPerSecond;
  size_t generation_delay_micros =
      flags.getInt("generation_delay_secs") * kMicrosPerSecond;

  FileUtil::mkdir_p(output_path);

  HashMap<String, uint64_t> feed_offsets;
  HashMap<uint64_t, std::unique_ptr<sstable::SSTableWriter>> generations;
  Set<uint64_t> finished_generations;
  uint64_t max_gen = 0;

  /* check old sstables */
  FileUtil::ls(output_path, [
      output_path,
      output_prefix,
      &max_gen,
      &feed_offsets,
      &finished_generations] (const String& filename) -> bool {
    if (!StringUtil::beginsWith(filename, output_prefix) ||
        !StringUtil::endsWith(filename, ".sstable")) {
      return true;
    }

    auto filepath = FileUtil::joinPaths(output_path, filename);
    sstable::SSTableReader reader(
        File::openFile(filepath, File::O_READ));

    if (reader.bodySize() == 0) {
        fnord::logWarning(
            "fnord.feedexport",
            "Deleting unfinished sstable $0",
            filepath);

      FileUtil::rm(filepath);
      return true;
    }

    auto hdr = reader.readHeader();
    auto fci = json::fromJSON<FeedChunkInfo>(hdr);

    for (const auto& fo : fci.start_offsets) {
      auto foff = std::stoul(fo.second);
      if (feed_offsets[fo.first] < foff) {
        feed_offsets[fo.first] = foff;
      }
    }

    if (fci.generation > max_gen) {
      max_gen = fci.generation;
    }

    finished_generations.emplace(fci.generation);
    return true;
  });

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);

  /* get source urls */
  Vector<String> uris = flags.getArgv();
  HashMap<String, String> feed_urls;
  for (const auto& uri_raw : uris) {
    URI uri(uri_raw);
    const auto& params = uri.queryParams();

    std::string feed;
    if (!URI::getParam(params, "feed", &feed)) {
      RAISEF(
          kIllegalArgumentError,
          "feed url missing ?feed query param: $0",
          uri_raw);
    }

    feed_urls.emplace(feed, uri_raw.substr(0, uri_raw.find("?")));

    uint64_t offset = feed_offsets[feed];
    std::string offset_str;
    if (URI::getParam(params, "offset", &offset_str)) {
      if (offset_str == "HEAD") {
        offset = std::numeric_limits<uint64_t>::max();
      } else {
        offset = std::stoul(offset_str);
      }
    }

    feed_reader.addSourceFeed(
        uri,
        feed,
        offset,
        batch_size,
        buffer_size);
  }

  uint64_t total_rows = 0;
  uint64_t total_rows_written = 0;
  uint64_t total_bytes = 0;
  auto start_time = WallClock::now().unixMicros();
  auto last_status_line = start_time;

  DateTime last_iter;
  uint64_t rate_limit_micros = 0.1 * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();

    /* read feed and build generations */
    int i = 0;
    for (; i < batch_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      ++total_rows;
      total_bytes += entry.get().data.size();

      auto now = WallClock::now().unixMicros();
      if (now - last_status_line > 10000) {
        Set<uint64_t> active_gens;
        for (const auto& g : generations) {
          active_gens.emplace(g.first);
        }

        auto watermarks = feed_reader.watermarks();
        auto runtime = (now - start_time) / 1000000;
        uint64_t bandwidth = total_bytes / (runtime + 1);
        auto str = StringUtil::format(
            "\rrows=$0 bytes=$1B time=$2s bw=$3B/s active_gens=$4 streamtime=$5"
            "          ",
            total_rows,
            total_bytes,
            runtime,
            bandwidth,
            inspect(active_gens),
            watermarks.first);

        write(0, str.c_str(), str.length());
        fflush(0);
      }

      auto entry_time = entry.get().time.unixMicros();
      if (entry_time == 0) {
        continue;
      }

      uint64_t entry_gen = entry_time / generation_window_micros;

      auto iter = generations.find(entry_gen);
      if (iter == generations.end()) {
        if (max_gen >= entry_gen) {
          if (finished_generations.count(entry_gen) == 0) {
            fnord::logWarning(
                "fnord.feedexport",
                "Dropping late data for  generation #$0",
                entry_gen);
          }

          continue;
        }

        fnord::logDebug(
            "fnord.feedexport",
            "Creating new generation #$0",
            entry_gen);

        FeedChunkInfo fci;
        fci.generation = entry_gen;
        fci.generation_window = generation_window_micros;
        fci.start_time = entry_gen * generation_window_micros;
        fci.end_time = (entry_gen + 1) * generation_window_micros;

        auto stream_offsets = feed_reader.streamOffsets();
        for (const auto& soff : stream_offsets) {
          fci.start_offsets.emplace(
              soff.first,
              StringUtil::toString(soff.second));
        }

        auto fci_json = json::toJSONString(fci);

        auto sstable_file_path = FileUtil::joinPaths(
            output_path,
            StringUtil::format("$0.$1.sstable", output_prefix, entry_gen));

        auto sstable_writer = sstable::SSTableWriter::create(
            sstable_file_path,
            sstable::IndexProvider{},
            fci_json.c_str(),
            fci_json.length());

        if (entry_gen > max_gen) {
          max_gen = entry_gen;
        }

        generations.emplace(entry_gen, std::move(sstable_writer));
      }

      const auto& entry_data = entry.get().data;
      generations[entry_gen]->appendRow(
          (void *) &entry_time,
          sizeof(entry_time),
          entry_data.c_str(),
          entry_data.length());

      ++total_rows_written;
    }

    feed_reader.fillBuffers();

    /* flush generations */
    auto watermarks = feed_reader.watermarks();
    auto stream_time_micros = watermarks.first.unixMicros();

    if (stream_time_micros > generation_delay_micros) {
      stream_time_micros -= generation_delay_micros;
    }

    for (auto iter = generations.begin(); iter != generations.end(); ) {
      if (stream_time_micros < ((iter->first + 1) * generation_window_micros)) {
        ++iter;
        continue;
      }

      fnord::logDebug(
          "cm.ctrstatsbuild",
          "Flushing report generation #$0",
          iter->first);

      iter->second->finalize();
      iter = generations.erase(iter);
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  evloop_thread.join();
  return 0;
}

<commit_msg>allow for up to generation_delay / 2 secs spread<commit_after>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 Paul Asmuth, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stdlib.h>
#include <unistd.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-mdb/MDB.h"
#include "cm-common/CustomerNamespace.h"
#include "cm-logjoin/LogJoin.h"

using namespace fnord;

struct FeedChunkInfo {
  uint64_t generation;
  uint64_t generation_window;
  uint64_t start_time;
  uint64_t end_time;
  HashMap<String, String> start_offsets;

  template <typename T>
  static void reflect(T* meta) {
    meta->prop(&FeedChunkInfo::generation, 1, "generation", false);
    meta->prop(&FeedChunkInfo::generation_window, 2, "generation_window", false);
    meta->prop(&FeedChunkInfo::start_time, 3, "start_time", false);
    meta->prop(&FeedChunkInfo::end_time, 4, "end_time", false);
    meta->prop(&FeedChunkInfo::start_offsets, 5, "start_offsets", false);
  };
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_path",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "output_prefix",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output filename prefix",
      "<prefix>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "generation_window_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "14400",
      "generation_window_secs",
      "<secs>");

  flags.defineFlag(
      "generation_delay_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "3600",
      "generation_delay_secs",
      "<secs>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  auto output_path = flags.getString("output_path");
  auto output_prefix = flags.getString("output_prefix");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t generation_window_micros =
      flags.getInt("generation_window_secs") * kMicrosPerSecond;
  size_t generation_delay_micros =
      flags.getInt("generation_delay_secs") * kMicrosPerSecond;

  FileUtil::mkdir_p(output_path);

  HashMap<String, uint64_t> feed_offsets;
  HashMap<uint64_t, std::unique_ptr<sstable::SSTableWriter>> generations;
  Set<uint64_t> finished_generations;
  uint64_t max_gen = 0;

  /* check old sstables */
  FileUtil::ls(output_path, [
      output_path,
      output_prefix,
      &max_gen,
      &feed_offsets,
      &finished_generations] (const String& filename) -> bool {
    if (!StringUtil::beginsWith(filename, output_prefix) ||
        !StringUtil::endsWith(filename, ".sstable")) {
      return true;
    }

    auto filepath = FileUtil::joinPaths(output_path, filename);
    sstable::SSTableReader reader(
        File::openFile(filepath, File::O_READ));

    if (reader.bodySize() == 0) {
        fnord::logWarning(
            "fnord.feedexport",
            "Deleting unfinished sstable $0",
            filepath);

      FileUtil::rm(filepath);
      return true;
    }

    auto hdr = reader.readHeader();
    auto fci = json::fromJSON<FeedChunkInfo>(hdr);

    for (const auto& fo : fci.start_offsets) {
      auto foff = std::stoul(fo.second);
      if (feed_offsets[fo.first] < foff) {
        feed_offsets[fo.first] = foff;
      }
    }

    if (fci.generation > max_gen) {
      max_gen = fci.generation;
    }

    finished_generations.emplace(fci.generation);
    return true;
  });

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(generation_delay_micros / 2);

  /* get source urls */
  Vector<String> uris = flags.getArgv();
  HashMap<String, String> feed_urls;
  for (const auto& uri_raw : uris) {
    URI uri(uri_raw);
    const auto& params = uri.queryParams();

    std::string feed;
    if (!URI::getParam(params, "feed", &feed)) {
      RAISEF(
          kIllegalArgumentError,
          "feed url missing ?feed query param: $0",
          uri_raw);
    }

    feed_urls.emplace(feed, uri_raw.substr(0, uri_raw.find("?")));

    uint64_t offset = feed_offsets[feed];
    std::string offset_str;
    if (URI::getParam(params, "offset", &offset_str)) {
      if (offset_str == "HEAD") {
        offset = std::numeric_limits<uint64_t>::max();
      } else {
        offset = std::stoul(offset_str);
      }
    }

    feed_reader.addSourceFeed(
        uri,
        feed,
        offset,
        batch_size,
        buffer_size);
  }

  uint64_t total_rows = 0;
  uint64_t total_rows_written = 0;
  uint64_t total_bytes = 0;
  auto start_time = WallClock::now().unixMicros();
  auto last_status_line = start_time;

  DateTime last_iter;
  uint64_t rate_limit_micros = 0.1 * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();

    /* read feed and build generations */
    int i = 0;
    for (; i < batch_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      ++total_rows;
      total_bytes += entry.get().data.size();

      auto now = WallClock::now().unixMicros();
      if (now - last_status_line > 10000) {
        Set<uint64_t> active_gens;
        for (const auto& g : generations) {
          active_gens.emplace(g.first);
        }

        auto watermarks = feed_reader.watermarks();
        auto runtime = (now - start_time) / 1000000;
        uint64_t bandwidth = total_bytes / (runtime + 1);
        auto str = StringUtil::format(
            "\rrows=$0 bytes=$1B time=$2s bw=$3B/s active_gens=$4 streamtime=$5"
            "          ",
            total_rows,
            total_bytes,
            runtime,
            bandwidth,
            inspect(active_gens),
            watermarks.first);

        write(0, str.c_str(), str.length());
        fflush(0);
      }

      auto entry_time = entry.get().time.unixMicros();
      if (entry_time == 0) {
        continue;
      }

      uint64_t entry_gen = entry_time / generation_window_micros;

      auto iter = generations.find(entry_gen);
      if (iter == generations.end()) {
        if (max_gen >= entry_gen) {
          if (finished_generations.count(entry_gen) == 0) {
            fnord::logWarning(
                "fnord.feedexport",
                "Dropping late data for  generation #$0",
                entry_gen);
          }

          continue;
        }

        fnord::logDebug(
            "fnord.feedexport",
            "Creating new generation #$0",
            entry_gen);

        FeedChunkInfo fci;
        fci.generation = entry_gen;
        fci.generation_window = generation_window_micros;
        fci.start_time = entry_gen * generation_window_micros;
        fci.end_time = (entry_gen + 1) * generation_window_micros;

        auto stream_offsets = feed_reader.streamOffsets();
        for (const auto& soff : stream_offsets) {
          fci.start_offsets.emplace(
              soff.first,
              StringUtil::toString(soff.second));
        }

        auto fci_json = json::toJSONString(fci);

        auto sstable_file_path = FileUtil::joinPaths(
            output_path,
            StringUtil::format("$0.$1.sstable", output_prefix, entry_gen));

        auto sstable_writer = sstable::SSTableWriter::create(
            sstable_file_path,
            sstable::IndexProvider{},
            fci_json.c_str(),
            fci_json.length());

        if (entry_gen > max_gen) {
          max_gen = entry_gen;
        }

        generations.emplace(entry_gen, std::move(sstable_writer));
      }

      const auto& entry_data = entry.get().data;
      generations[entry_gen]->appendRow(
          (void *) &entry_time,
          sizeof(entry_time),
          entry_data.c_str(),
          entry_data.length());

      ++total_rows_written;
    }

    feed_reader.fillBuffers();

    /* flush generations */
    auto watermarks = feed_reader.watermarks();
    auto stream_time_micros = watermarks.first.unixMicros();

    if (stream_time_micros > generation_delay_micros) {
      stream_time_micros -= generation_delay_micros;
    }

    for (auto iter = generations.begin(); iter != generations.end(); ) {
      if (stream_time_micros < ((iter->first + 1) * generation_window_micros)) {
        ++iter;
        continue;
      }

      fnord::logDebug(
          "cm.ctrstatsbuild",
          "Flushing report generation #$0",
          iter->first);

      iter->second->finalize();
      iter = generations.erase(iter);
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  evloop_thread.join();
  return 0;
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/Random.h>

#include <array>
#include <mutex>
#include <random>

#include <folly/CppAttributes.h>
#include <folly/File.h>
#include <folly/FileUtil.h>
#include <folly/SingletonThreadLocal.h>
#include <folly/ThreadLocal.h>
#include <folly/detail/FileUtilDetail.h>
#include <folly/portability/Config.h>
#include <folly/portability/SysTime.h>
#include <folly/portability/Unistd.h>
#include <folly/synchronization/RelaxedAtomic.h>

#include <glog/logging.h>

#ifdef _MSC_VER
#include <wincrypt.h> // @manual
#endif

#if FOLLY_HAVE_GETRANDOM
#include <sys/random.h>
#endif

namespace folly {

namespace {

void readRandomDevice(void* data, size_t size) {
#ifdef _MSC_VER
  static auto const cryptoProv = [] {
    HCRYPTPROV prov;
    if (!CryptAcquireContext(
            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) {
      if (GetLastError() == NTE_BAD_KEYSET) {
        // Mostly likely cause of this is that no key container
        // exists yet, so try to create one.
        PCHECK(CryptAcquireContext(
            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_NEWKEYSET));
      } else {
        LOG(FATAL) << "Failed to acquire the default crypto context.";
      }
    }
    return prov;
  }();
  CHECK(size <= std::numeric_limits<DWORD>::max());
  PCHECK(CryptGenRandom(cryptoProv, (DWORD)size, (BYTE*)data));
#else
  ssize_t bytesRead = 0;
  auto gen = [](int, void* buf, size_t buflen) {
#if FOLLY_HAVE_GETRANDOM
    auto flags = 0u;
    return ::getrandom(buf, buflen, flags);
#else
    [](...) {}(buf, buflen);
    errno = ENOSYS;
    return -1;
#endif
  };
  bytesRead = fileutil_detail::wrapFull(gen, -1, data, size);
  if (bytesRead == -1 && errno == ENOSYS) {
    // Keep the random device open for the duration of the program.
    static int randomFd = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC);
    PCHECK(randomFd >= 0);
    bytesRead = readFull(randomFd, data, size);
  }
  PCHECK(bytesRead >= 0);
  CHECK_EQ(size_t(bytesRead), size);
#endif
}

class BufferedRandomDevice {
 public:
  static constexpr size_t kDefaultBufferSize = 128;

  static void notifyNewGlobalEpoch() { ++globalEpoch_; }

  explicit BufferedRandomDevice(size_t bufferSize = kDefaultBufferSize);

  void get(void* data, size_t size) {
    if (LIKELY(epoch_ == globalEpoch_ && size <= remaining())) {
      memcpy(data, ptr_, size);
      ptr_ += size;
    } else {
      getSlow(static_cast<unsigned char*>(data), size);
    }
  }

 private:
  void getSlow(unsigned char* data, size_t size);

  inline size_t remaining() const {
    return size_t(buffer_.get() + bufferSize_ - ptr_);
  }

  static relaxed_atomic<size_t> globalEpoch_;

  size_t epoch_{size_t(-1)}; // refill on first use
  const size_t bufferSize_;
  std::unique_ptr<unsigned char[]> buffer_;
  unsigned char* ptr_;
};

relaxed_atomic<size_t> BufferedRandomDevice::globalEpoch_{0};
struct RandomTag {};

BufferedRandomDevice::BufferedRandomDevice(size_t bufferSize)
    : bufferSize_(bufferSize),
      buffer_(new unsigned char[bufferSize]),
      ptr_(buffer_.get() + bufferSize) { // refill on first use
  FOLLY_MAYBE_UNUSED static auto const init = [] {
    detail::AtFork::registerHandler(
        nullptr,
        /*prepare*/ []() { return true; },
        /*parent*/ []() {},
        /*child*/
        []() {
          // Ensure child and parent do not share same entropy pool.
          BufferedRandomDevice::notifyNewGlobalEpoch();
        });
    return 0;
  }();
}

void BufferedRandomDevice::getSlow(unsigned char* data, size_t size) {
  if (epoch_ != globalEpoch_) {
    epoch_ = globalEpoch_;
    ptr_ = buffer_.get() + bufferSize_;
  }

  DCHECK_GT(size, remaining());
  if (size >= bufferSize_) {
    // Just read directly.
    readRandomDevice(data, size);
    return;
  }

  size_t copied = remaining();
  memcpy(data, ptr_, copied);
  data += copied;
  size -= copied;

  // refill
  readRandomDevice(buffer_.get(), bufferSize_);
  ptr_ = buffer_.get();

  memcpy(data, ptr_, size);
  ptr_ += size;
}

} // namespace

void Random::secureRandom(void* data, size_t size) {
  using Single = SingletonThreadLocal<BufferedRandomDevice, RandomTag>;
  Single::get().get(data, size);
}

ThreadLocalPRNG::result_type ThreadLocalPRNG::operator()() {
  struct Wrapper {
    Random::DefaultGenerator object{Random::create()};
  };
  using Single = SingletonThreadLocal<Wrapper, RandomTag>;
  return Single::get().object();
}
} // namespace folly
<commit_msg>cut dep from Random onto File and FileUtil<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/Random.h>

#include <array>
#include <mutex>
#include <random>

#include <folly/CppAttributes.h>
#include <folly/SingletonThreadLocal.h>
#include <folly/ThreadLocal.h>
#include <folly/detail/FileUtilDetail.h>
#include <folly/portability/Config.h>
#include <folly/portability/SysTime.h>
#include <folly/portability/Unistd.h>
#include <folly/synchronization/RelaxedAtomic.h>

#include <glog/logging.h>

#ifdef _MSC_VER
#include <wincrypt.h> // @manual
#else
#include <fcntl.h>
#endif

#if FOLLY_HAVE_GETRANDOM
#include <sys/random.h>
#endif

namespace folly {

namespace {

void readRandomDevice(void* data, size_t size) {
#ifdef _MSC_VER
  static auto const cryptoProv = [] {
    HCRYPTPROV prov;
    if (!CryptAcquireContext(
            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) {
      if (GetLastError() == NTE_BAD_KEYSET) {
        // Mostly likely cause of this is that no key container
        // exists yet, so try to create one.
        PCHECK(CryptAcquireContext(
            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_NEWKEYSET));
      } else {
        LOG(FATAL) << "Failed to acquire the default crypto context.";
      }
    }
    return prov;
  }();
  CHECK(size <= std::numeric_limits<DWORD>::max());
  PCHECK(CryptGenRandom(cryptoProv, (DWORD)size, (BYTE*)data));
#else
  ssize_t bytesRead = 0;
  auto gen = [](int, void* buf, size_t buflen) {
#if FOLLY_HAVE_GETRANDOM
    auto flags = 0u;
    return ::getrandom(buf, buflen, flags);
#else
    [](...) {}(buf, buflen);
    errno = ENOSYS;
    return -1;
#endif
  };
  bytesRead = fileutil_detail::wrapFull(gen, -1, data, size);
  if (bytesRead == -1 && errno == ENOSYS) {
    // Keep the random device open for the duration of the program.
    static int randomFd = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC);
    PCHECK(randomFd >= 0);
    bytesRead = fileutil_detail::wrapFull(::read, randomFd, data, size);
  }
  PCHECK(bytesRead >= 0);
  CHECK_EQ(size_t(bytesRead), size);
#endif
}

class BufferedRandomDevice {
 public:
  static constexpr size_t kDefaultBufferSize = 128;

  static void notifyNewGlobalEpoch() { ++globalEpoch_; }

  explicit BufferedRandomDevice(size_t bufferSize = kDefaultBufferSize);

  void get(void* data, size_t size) {
    if (LIKELY(epoch_ == globalEpoch_ && size <= remaining())) {
      memcpy(data, ptr_, size);
      ptr_ += size;
    } else {
      getSlow(static_cast<unsigned char*>(data), size);
    }
  }

 private:
  void getSlow(unsigned char* data, size_t size);

  inline size_t remaining() const {
    return size_t(buffer_.get() + bufferSize_ - ptr_);
  }

  static relaxed_atomic<size_t> globalEpoch_;

  size_t epoch_{size_t(-1)}; // refill on first use
  const size_t bufferSize_;
  std::unique_ptr<unsigned char[]> buffer_;
  unsigned char* ptr_;
};

relaxed_atomic<size_t> BufferedRandomDevice::globalEpoch_{0};
struct RandomTag {};

BufferedRandomDevice::BufferedRandomDevice(size_t bufferSize)
    : bufferSize_(bufferSize),
      buffer_(new unsigned char[bufferSize]),
      ptr_(buffer_.get() + bufferSize) { // refill on first use
  FOLLY_MAYBE_UNUSED static auto const init = [] {
    detail::AtFork::registerHandler(
        nullptr,
        /*prepare*/ []() { return true; },
        /*parent*/ []() {},
        /*child*/
        []() {
          // Ensure child and parent do not share same entropy pool.
          BufferedRandomDevice::notifyNewGlobalEpoch();
        });
    return 0;
  }();
}

void BufferedRandomDevice::getSlow(unsigned char* data, size_t size) {
  if (epoch_ != globalEpoch_) {
    epoch_ = globalEpoch_;
    ptr_ = buffer_.get() + bufferSize_;
  }

  DCHECK_GT(size, remaining());
  if (size >= bufferSize_) {
    // Just read directly.
    readRandomDevice(data, size);
    return;
  }

  size_t copied = remaining();
  memcpy(data, ptr_, copied);
  data += copied;
  size -= copied;

  // refill
  readRandomDevice(buffer_.get(), bufferSize_);
  ptr_ = buffer_.get();

  memcpy(data, ptr_, size);
  ptr_ += size;
}

} // namespace

void Random::secureRandom(void* data, size_t size) {
  using Single = SingletonThreadLocal<BufferedRandomDevice, RandomTag>;
  Single::get().get(data, size);
}

ThreadLocalPRNG::result_type ThreadLocalPRNG::operator()() {
  struct Wrapper {
    Random::DefaultGenerator object{Random::create()};
  };
  using Single = SingletonThreadLocal<Wrapper, RandomTag>;
  return Single::get().object();
}
} // namespace folly
<|endoftext|>


<commit_before><commit_msg>also backport 1.881 for vertical scrolling of checkboxes, thanks Sandro.<commit_after><|endoftext|>


<commit_before><commit_msg>Enable completion for the reply-to field.<commit_after><|endoftext|>


<commit_before>/*
    Kopete Yahoo Protocol
    Notifies about incoming filetransfers

    Copyright (c) 2006 André Duffeck <andre.duffeck@kdemail.net>

    *************************************************************************
    *                                                                       *
    * This library is free software; you can redistribute it and/or         *
    * modify it under the terms of the GNU Lesser General Public            *
    * License as published by the Free Software Foundation; either          *
    * version 2 of the License, or (at your option) any later version.      *
    *                                                                       *
    *************************************************************************
*/

#include "filetransfernotifiertask.h"
#include "transfer.h"
#include "ymsgtransfer.h"
#include "yahootypes.h"
#include "client.h"
#include <qstring.h>
#include <kdebug.h>

FileTransferNotifierTask::FileTransferNotifierTask(Task* parent) : Task(parent)
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;
}

FileTransferNotifierTask::~FileTransferNotifierTask()
{

}

bool FileTransferNotifierTask::take( Transfer* transfer )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	if ( !forMe( transfer ) )
		return false;

	YMSGTransfer *t = static_cast<YMSGTransfer*>(transfer);

	if( t->service() == Yahoo::ServiceFileTransfer )
		parseFileTransfer( t );
	else if( t->service() == Yahoo::ServiceFileTransfer7 )
		parseFileTransfer7( t );
	else if( t->service() == Yahoo::ServicePeerToPeer )
		acceptFileTransfer( t );

	return true;
}

bool FileTransferNotifierTask::forMe( Transfer *transfer ) const
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;
	YMSGTransfer *t = 0L;
	t = dynamic_cast<YMSGTransfer*>(transfer);
	if (!t)
		return false;

	if( t->service() == Yahoo::ServiceP2PFileXfer ||
		t->service() == Yahoo::ServicePeerToPeer ||
		t->service() == Yahoo::ServiceFileTransfer ||
		t->service() == Yahoo::ServiceFileTransfer7
	)
		return true;
	else
		return false;
}

void FileTransferNotifierTask::parseFileTransfer( YMSGTransfer *t )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	QString from;		/* key = 4  */
	QString to;		/* key = 5  */
	QString url;		/* key = 20  */
	long expires;		/* key = 38  */
	QString msg;		/* key = 14  */
	QString filename;	/* key = 27  */
	unsigned long size;	/* key = 28  */

	from = t->firstParam( 4 );
	to = t->firstParam( 5 );
	url = t->firstParam( 20 );
	expires = t->firstParam( 38 ).toLong();
	msg = t->firstParam( 14 );
	filename = t->firstParam( 27 );
	size = t->firstParam( 28 ).toULong();

	if( from.startsWith( "FILE_TRANSFER_SYSTEM" ) )
	{
		client()->notifyError( "Fileupload result received.", msg, Client::Notice );
	}

	if( url.isEmpty() )
		return;

	unsigned int left = url.findRev( '/' ) + 1;
	unsigned int right = url.findRev( '?' );
	filename = url.mid( left, right - left );

	emit incomingFileTransfer( from, url, expires, msg, filename, size );
}

void FileTransferNotifierTask::parseFileTransfer7( YMSGTransfer *t )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	QString from;		/* key = 4  */
	QString to;		/* key = 5  */
	QString url;		/* key = 20  */
	long expires;		/* key = 38  */
	QString msg;		/* key = 14  */
	QString filename;	/* key = 27  */
	unsigned long size;	/* key = 28  */

	if( t->firstParam( 222 ).toInt() == 2 )
		return;					// user cancelled the file transfer

	from = t->firstParam( 4 );
	to = t->firstParam( 5 );
	url = t->firstParam( 265 );
	msg = t->firstParam( 14 );
	filename = t->firstParam( 27 );
	size = t->firstParam( 28 ).toULong();

	emit incomingFileTransfer( from, url, expires, msg, filename, size );
}

void FileTransferNotifierTask::acceptFileTransfer( YMSGTransfer *transfer )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	YMSGTransfer *t = new YMSGTransfer(Yahoo::ServicePeerToPeer);
	t->setId( client()->sessionID() );
	t->setParam( 4, client()->userId().local8Bit() );
	t->setParam( 5, transfer->firstParam( 4 ) );
	t->setParam( 11, transfer->firstParam( 11 ) );

	send( t );
}

#include "filetransfernotifiertask.moc"
<commit_msg>return in case of a fileupload result<commit_after>/*
    Kopete Yahoo Protocol
    Notifies about incoming filetransfers

    Copyright (c) 2006 André Duffeck <andre.duffeck@kdemail.net>

    *************************************************************************
    *                                                                       *
    * This library is free software; you can redistribute it and/or         *
    * modify it under the terms of the GNU Lesser General Public            *
    * License as published by the Free Software Foundation; either          *
    * version 2 of the License, or (at your option) any later version.      *
    *                                                                       *
    *************************************************************************
*/

#include "filetransfernotifiertask.h"
#include "transfer.h"
#include "ymsgtransfer.h"
#include "yahootypes.h"
#include "client.h"
#include <qstring.h>
#include <kdebug.h>

FileTransferNotifierTask::FileTransferNotifierTask(Task* parent) : Task(parent)
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;
}

FileTransferNotifierTask::~FileTransferNotifierTask()
{

}

bool FileTransferNotifierTask::take( Transfer* transfer )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	if ( !forMe( transfer ) )
		return false;

	YMSGTransfer *t = static_cast<YMSGTransfer*>(transfer);

	if( t->service() == Yahoo::ServiceFileTransfer )
		parseFileTransfer( t );
	else if( t->service() == Yahoo::ServiceFileTransfer7 )
		parseFileTransfer7( t );
	else if( t->service() == Yahoo::ServicePeerToPeer )
		acceptFileTransfer( t );

	return true;
}

bool FileTransferNotifierTask::forMe( Transfer *transfer ) const
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;
	YMSGTransfer *t = 0L;
	t = dynamic_cast<YMSGTransfer*>(transfer);
	if (!t)
		return false;

	if( t->service() == Yahoo::ServiceP2PFileXfer ||
		t->service() == Yahoo::ServicePeerToPeer ||
		t->service() == Yahoo::ServiceFileTransfer ||
		t->service() == Yahoo::ServiceFileTransfer7
	)
		return true;
	else
		return false;
}

void FileTransferNotifierTask::parseFileTransfer( YMSGTransfer *t )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	QString from;		/* key = 4  */
	QString to;		/* key = 5  */
	QString url;		/* key = 20  */
	long expires;		/* key = 38  */
	QString msg;		/* key = 14  */
	QString filename;	/* key = 27  */
	unsigned long size;	/* key = 28  */

	from = t->firstParam( 4 );
	to = t->firstParam( 5 );
	url = t->firstParam( 20 );
	expires = t->firstParam( 38 ).toLong();
	msg = t->firstParam( 14 );
	filename = t->firstParam( 27 );
	size = t->firstParam( 28 ).toULong();

	if( from.startsWith( "FILE_TRANSFER_SYSTEM" ) )
	{
		client()->notifyError( "Fileupload result received.", msg, Client::Notice );
		return;
	}

	if( url.isEmpty() )
		return;

	unsigned int left = url.findRev( '/' ) + 1;
	unsigned int right = url.findRev( '?' );
	filename = url.mid( left, right - left );

	emit incomingFileTransfer( from, url, expires, msg, filename, size );
}

void FileTransferNotifierTask::parseFileTransfer7( YMSGTransfer *t )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	QString from;		/* key = 4  */
	QString to;		/* key = 5  */
	QString url;		/* key = 20  */
	long expires;		/* key = 38  */
	QString msg;		/* key = 14  */
	QString filename;	/* key = 27  */
	unsigned long size;	/* key = 28  */

	if( t->firstParam( 222 ).toInt() == 2 )
		return;					// user cancelled the file transfer

	from = t->firstParam( 4 );
	to = t->firstParam( 5 );
	url = t->firstParam( 265 );
	msg = t->firstParam( 14 );
	filename = t->firstParam( 27 );
	size = t->firstParam( 28 ).toULong();

	emit incomingFileTransfer( from, url, expires, msg, filename, size );
}

void FileTransferNotifierTask::acceptFileTransfer( YMSGTransfer *transfer )
{
	kdDebug(YAHOO_RAW_DEBUG) << k_funcinfo << endl;

	YMSGTransfer *t = new YMSGTransfer(Yahoo::ServicePeerToPeer);
	t->setId( client()->sessionID() );
	t->setParam( 4, client()->userId().local8Bit() );
	t->setParam( 5, transfer->firstParam( 4 ) );
	t->setParam( 11, transfer->firstParam( 11 ) );

	send( t );
}

#include "filetransfernotifiertask.moc"
<|endoftext|>


<commit_before>/*
-----------------------------------------------------------------------------
This source file is part of OGRE
(Object-oriented Graphics Rendering Engine)
For the latest info, see http://www.ogre3d.org/

Copyright (c) 2000-2014 Torus Knot Software Ltd

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-----------------------------------------------------------------------------
*/

#include "OgreSTBICodec.h"
#include "OgreLogManager.h"
#include "OgreDataStream.h"

#include "OgrePlatformInformation.h"

#if __OGRE_HAVE_NEON
#define STBI_NEON
#endif

#define STBI_NO_STDIO
#define STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_STATIC
#include "stbi/stb_image.h"

#if OGRE_NO_ZIP_ARCHIVE == 0
#include <zlib.h>
static Ogre::uchar* custom_zlib_compress(Ogre::uchar* data, int data_len, int* out_len, int /*quality*/)
{
    unsigned long destLen = compressBound(data_len);
    Ogre::uchar* dest = (Ogre::uchar*)malloc(destLen);
    int ret = compress(dest, &destLen, data, data_len); // use default quality
    if (ret != Z_OK)
    {
        free(dest);
        OGRE_EXCEPT(Ogre::Exception::ERR_INTERNAL_ERROR, "compress failed");
    }

    *out_len = destLen;
    return dest;
}
#define STBIW_ZLIB_COMPRESS custom_zlib_compress
#endif

#define STB_IMAGE_WRITE_IMPLEMENTATION
#define STBI_WRITE_NO_STDIO
#include "stbi/stb_image_write.h"

namespace Ogre {

    STBIImageCodec::RegisteredCodecList STBIImageCodec::msCodecList;
    //---------------------------------------------------------------------
    void STBIImageCodec::startup(void)
    {
        stbi_convert_iphone_png_to_rgb(1);
        stbi_set_unpremultiply_on_load(1);

        LogManager::getSingleton().logMessage("stb_image - v2.23 - public domain image loader");

        // Register codecs
        String exts = "jpeg,jpg,png,bmp,psd,tga,gif,pic,ppm,pgm,hdr";
        StringVector extsVector = StringUtil::split(exts, ",");
        for (StringVector::iterator v = extsVector.begin(); v != extsVector.end(); ++v)
        {
            ImageCodec* codec = OGRE_NEW STBIImageCodec(*v);
            msCodecList.push_back(codec);
            Codec::registerCodec(codec);
        }

        LogManager::getSingleton().logMessage("Supported formats: " + exts);
    }
    //---------------------------------------------------------------------
    void STBIImageCodec::shutdown(void)
    {
        for (RegisteredCodecList::iterator i = msCodecList.begin();
            i != msCodecList.end(); ++i)
        {
            Codec::unregisterCodec(*i);
            OGRE_DELETE *i;
        }
        msCodecList.clear();
    }
    //---------------------------------------------------------------------
    STBIImageCodec::STBIImageCodec(const String &type):
        mType(type)
    {
    }
    //---------------------------------------------------------------------
    DataStreamPtr STBIImageCodec::encode(const MemoryDataStreamPtr& input,
                                         const Codec::CodecDataPtr& pData) const
    {
        if(mType != "png") {
            OGRE_EXCEPT(Exception::ERR_NOT_IMPLEMENTED,
                        "currently only encoding to PNG supported",
                        "STBIImageCodec::encode" ) ;
        }

        ImageData* pImgData = static_cast<ImageData*>(pData.get());
        PixelFormat format = pImgData->format;
        uchar* inputData = input->getPtr();

        // Convert image data to ABGR format for STBI (unless it's already compatible)
        uchar* tempData = 0;
        if(format != Ogre::PF_A8B8G8R8 && format != PF_B8G8R8 && format != PF_BYTE_LA &&
            format != PF_L8 && format != PF_R8)
        {
            format = Ogre::PF_A8B8G8R8;
            size_t tempDataSize = pImgData->width * pImgData->height * pImgData->depth * Ogre::PixelUtil::getNumElemBytes(format);
            tempData = OGRE_ALLOC_T(unsigned char, tempDataSize, Ogre::MEMCATEGORY_GENERAL);
            Ogre::PixelBox pbIn(pImgData->width, pImgData->height, pImgData->depth, pImgData->format, inputData);
            Ogre::PixelBox pbOut(pImgData->width, pImgData->height, pImgData->depth, format, tempData);
            PixelUtil::bulkPixelConversion(pbIn, pbOut);

            inputData = tempData;
        }

        // Save to PNG
        int channels = (int)PixelUtil::getComponentCount(format);
        int stride = pImgData->width * (int)PixelUtil::getNumElemBytes(format);
        int len;
        uchar* data = stbi_write_png_to_mem(inputData, stride, pImgData->width, pImgData->height, channels, &len);

        if(tempData)
        {
            OGRE_FREE(tempData, MEMCATEGORY_GENERAL);
        }

        if (!data) {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR,
                "Error encoding image: " + String(stbi_failure_reason()),
                "STBIImageCodec::encode");
        }

        return DataStreamPtr(new MemoryDataStream(data, len, true));
    }
    //---------------------------------------------------------------------
    void STBIImageCodec::encodeToFile(const MemoryDataStreamPtr& input, const String& outFileName,
                                      const Codec::CodecDataPtr& pData) const
    {
#if OGRE_PLATFORM != OGRE_PLATFORM_EMSCRIPTEN
        MemoryDataStreamPtr data = static_pointer_cast<MemoryDataStream>(encode(input, pData));
        std::ofstream f(outFileName.c_str(), std::ios::out | std::ios::binary);

        if(!f.is_open()) {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR,
                        "could not open file",
                        "STBIImageCodec::encodeToFile" ) ;
        }

        f.write((char*)data->getPtr(), data->size());
#endif
    }
    //---------------------------------------------------------------------
    Codec::DecodeResult STBIImageCodec::decode(const DataStreamPtr& input) const
    {
        String contents = input->getAsString();

        int width, height, components;
        stbi_uc* pixelData = stbi_load_from_memory((const uchar*)contents.data(),
                static_cast<int>(contents.size()), &width, &height, &components, 0);

        if (!pixelData)
        {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR,
                "Error decoding image: " + String(stbi_failure_reason()),
                "STBIImageCodec::decode");
        }

        SharedPtr<ImageData> imgData(OGRE_NEW ImageData());

        imgData->depth = 1; // only 2D formats handled by this codec
        imgData->width = width;
        imgData->height = height;
        imgData->num_mipmaps = 0; // no mipmaps in non-DDS
        imgData->flags = 0;

        switch( components )
        {
            case 1:
                imgData->format = PF_BYTE_L;
                break;
            case 2:
                imgData->format = PF_BYTE_LA;
                break;
            case 3:
                imgData->format = PF_BYTE_RGB;
                break;
            case 4:
                imgData->format = PF_BYTE_RGBA;
                break;
            default:
                stbi_image_free(pixelData);
                OGRE_EXCEPT(Exception::ERR_ITEM_NOT_FOUND,
                            "Unknown or unsupported image format",
                            "STBIImageCodec::decode");
                break;
        }

        size_t dstPitch = imgData->width * PixelUtil::getNumElemBytes(imgData->format);
        imgData->size = dstPitch * imgData->height;
        MemoryDataStreamPtr output(OGRE_NEW MemoryDataStream(pixelData, imgData->size, true));

        DecodeResult ret;
        ret.first = output;
        ret.second = imgData;
        return ret;
    }
    //---------------------------------------------------------------------
    String STBIImageCodec::getType() const
    {
        return mType;
    }
    //---------------------------------------------------------------------
    String STBIImageCodec::magicNumberToFileExt(const char *magicNumberPtr, size_t maxbytes) const
    {
        return BLANKSTRING;
    }

    const String& STBIPlugin::getName() const {
        static String name = "STB Image Codec";
        return name;
    }

#ifndef OGRE_STATIC_LIB
    extern "C" void _OgreSTBICodecExport dllStartPlugin();
    extern "C" void _OgreSTBICodecExport dllStopPlugin();

    extern "C" void _OgreSTBICodecExport dllStartPlugin()
    {
        STBIImageCodec::startup();
    }
    extern "C" void _OgreSTBICodecExport dllStopPlugin()
    {
        STBIImageCodec::shutdown();
    }
#endif
}
<commit_msg>STBICodec: encodeToFile - improve error message<commit_after>/*
-----------------------------------------------------------------------------
This source file is part of OGRE
(Object-oriented Graphics Rendering Engine)
For the latest info, see http://www.ogre3d.org/

Copyright (c) 2000-2014 Torus Knot Software Ltd

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-----------------------------------------------------------------------------
*/

#include "OgreSTBICodec.h"
#include "OgreLogManager.h"
#include "OgreDataStream.h"

#include "OgrePlatformInformation.h"

#if __OGRE_HAVE_NEON
#define STBI_NEON
#endif

#define STBI_NO_STDIO
#define STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_STATIC
#include "stbi/stb_image.h"

#if OGRE_NO_ZIP_ARCHIVE == 0
#include <zlib.h>
static Ogre::uchar* custom_zlib_compress(Ogre::uchar* data, int data_len, int* out_len, int /*quality*/)
{
    unsigned long destLen = compressBound(data_len);
    Ogre::uchar* dest = (Ogre::uchar*)malloc(destLen);
    int ret = compress(dest, &destLen, data, data_len); // use default quality
    if (ret != Z_OK)
    {
        free(dest);
        OGRE_EXCEPT(Ogre::Exception::ERR_INTERNAL_ERROR, "compress failed");
    }

    *out_len = destLen;
    return dest;
}
#define STBIW_ZLIB_COMPRESS custom_zlib_compress
#endif

#define STB_IMAGE_WRITE_IMPLEMENTATION
#define STBI_WRITE_NO_STDIO
#include "stbi/stb_image_write.h"

namespace Ogre {

    STBIImageCodec::RegisteredCodecList STBIImageCodec::msCodecList;
    //---------------------------------------------------------------------
    void STBIImageCodec::startup(void)
    {
        stbi_convert_iphone_png_to_rgb(1);
        stbi_set_unpremultiply_on_load(1);

        LogManager::getSingleton().logMessage("stb_image - v2.23 - public domain image loader");

        // Register codecs
        String exts = "jpeg,jpg,png,bmp,psd,tga,gif,pic,ppm,pgm,hdr";
        StringVector extsVector = StringUtil::split(exts, ",");
        for (StringVector::iterator v = extsVector.begin(); v != extsVector.end(); ++v)
        {
            ImageCodec* codec = OGRE_NEW STBIImageCodec(*v);
            msCodecList.push_back(codec);
            Codec::registerCodec(codec);
        }

        LogManager::getSingleton().logMessage("Supported formats: " + exts);
    }
    //---------------------------------------------------------------------
    void STBIImageCodec::shutdown(void)
    {
        for (RegisteredCodecList::iterator i = msCodecList.begin();
            i != msCodecList.end(); ++i)
        {
            Codec::unregisterCodec(*i);
            OGRE_DELETE *i;
        }
        msCodecList.clear();
    }
    //---------------------------------------------------------------------
    STBIImageCodec::STBIImageCodec(const String &type):
        mType(type)
    {
    }
    //---------------------------------------------------------------------
    DataStreamPtr STBIImageCodec::encode(const MemoryDataStreamPtr& input,
                                         const Codec::CodecDataPtr& pData) const
    {
        if(mType != "png") {
            OGRE_EXCEPT(Exception::ERR_NOT_IMPLEMENTED,
                        "currently only encoding to PNG supported",
                        "STBIImageCodec::encode" ) ;
        }

        ImageData* pImgData = static_cast<ImageData*>(pData.get());
        PixelFormat format = pImgData->format;
        uchar* inputData = input->getPtr();

        // Convert image data to ABGR format for STBI (unless it's already compatible)
        uchar* tempData = 0;
        if(format != Ogre::PF_A8B8G8R8 && format != PF_B8G8R8 && format != PF_BYTE_LA &&
            format != PF_L8 && format != PF_R8)
        {
            format = Ogre::PF_A8B8G8R8;
            size_t tempDataSize = pImgData->width * pImgData->height * pImgData->depth * Ogre::PixelUtil::getNumElemBytes(format);
            tempData = OGRE_ALLOC_T(unsigned char, tempDataSize, Ogre::MEMCATEGORY_GENERAL);
            Ogre::PixelBox pbIn(pImgData->width, pImgData->height, pImgData->depth, pImgData->format, inputData);
            Ogre::PixelBox pbOut(pImgData->width, pImgData->height, pImgData->depth, format, tempData);
            PixelUtil::bulkPixelConversion(pbIn, pbOut);

            inputData = tempData;
        }

        // Save to PNG
        int channels = (int)PixelUtil::getComponentCount(format);
        int stride = pImgData->width * (int)PixelUtil::getNumElemBytes(format);
        int len;
        uchar* data = stbi_write_png_to_mem(inputData, stride, pImgData->width, pImgData->height, channels, &len);

        if(tempData)
        {
            OGRE_FREE(tempData, MEMCATEGORY_GENERAL);
        }

        if (!data) {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR,
                "Error encoding image: " + String(stbi_failure_reason()),
                "STBIImageCodec::encode");
        }

        return DataStreamPtr(new MemoryDataStream(data, len, true));
    }
    //---------------------------------------------------------------------
    void STBIImageCodec::encodeToFile(const MemoryDataStreamPtr& input, const String& outFileName,
                                      const Codec::CodecDataPtr& pData) const
    {
#if OGRE_PLATFORM != OGRE_PLATFORM_EMSCRIPTEN
        MemoryDataStreamPtr data = static_pointer_cast<MemoryDataStream>(encode(input, pData));
        std::ofstream f(outFileName.c_str(), std::ios::out | std::ios::binary);

        if (!f.is_open())
        {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR, "could not open file " + outFileName);
        }

        f.write((char*)data->getPtr(), data->size());
#endif
    }
    //---------------------------------------------------------------------
    Codec::DecodeResult STBIImageCodec::decode(const DataStreamPtr& input) const
    {
        String contents = input->getAsString();

        int width, height, components;
        stbi_uc* pixelData = stbi_load_from_memory((const uchar*)contents.data(),
                static_cast<int>(contents.size()), &width, &height, &components, 0);

        if (!pixelData)
        {
            OGRE_EXCEPT(Exception::ERR_INTERNAL_ERROR,
                "Error decoding image: " + String(stbi_failure_reason()),
                "STBIImageCodec::decode");
        }

        SharedPtr<ImageData> imgData(OGRE_NEW ImageData());

        imgData->depth = 1; // only 2D formats handled by this codec
        imgData->width = width;
        imgData->height = height;
        imgData->num_mipmaps = 0; // no mipmaps in non-DDS
        imgData->flags = 0;

        switch( components )
        {
            case 1:
                imgData->format = PF_BYTE_L;
                break;
            case 2:
                imgData->format = PF_BYTE_LA;
                break;
            case 3:
                imgData->format = PF_BYTE_RGB;
                break;
            case 4:
                imgData->format = PF_BYTE_RGBA;
                break;
            default:
                stbi_image_free(pixelData);
                OGRE_EXCEPT(Exception::ERR_ITEM_NOT_FOUND,
                            "Unknown or unsupported image format",
                            "STBIImageCodec::decode");
                break;
        }

        size_t dstPitch = imgData->width * PixelUtil::getNumElemBytes(imgData->format);
        imgData->size = dstPitch * imgData->height;
        MemoryDataStreamPtr output(OGRE_NEW MemoryDataStream(pixelData, imgData->size, true));

        DecodeResult ret;
        ret.first = output;
        ret.second = imgData;
        return ret;
    }
    //---------------------------------------------------------------------
    String STBIImageCodec::getType() const
    {
        return mType;
    }
    //---------------------------------------------------------------------
    String STBIImageCodec::magicNumberToFileExt(const char *magicNumberPtr, size_t maxbytes) const
    {
        return BLANKSTRING;
    }

    const String& STBIPlugin::getName() const {
        static String name = "STB Image Codec";
        return name;
    }

#ifndef OGRE_STATIC_LIB
    extern "C" void _OgreSTBICodecExport dllStartPlugin();
    extern "C" void _OgreSTBICodecExport dllStopPlugin();

    extern "C" void _OgreSTBICodecExport dllStartPlugin()
    {
        STBIImageCodec::startup();
    }
    extern "C" void _OgreSTBICodecExport dllStopPlugin()
    {
        STBIImageCodec::shutdown();
    }
#endif
}
<|endoftext|>


<commit_before>#include "cubeb/cubeb.h"
#include <atomic>
#include <cassert>
#include <cmath>
#include <cstdarg>
#include <cstring>
#include <iostream>
#ifdef _WIN32
#include <objbase.h> // Used by CoInitialize()
#endif

#ifndef M_PI
#define M_PI 3.14159263
#endif

// Default values if none specified
#define DEFAULT_RATE 44100
#define DEFAULT_CHANNELS 2

static const char* state_to_string(cubeb_state state) {
  switch (state) {
    case CUBEB_STATE_STARTED:
      return "CUBEB_STATE_STARTED";
    case CUBEB_STATE_STOPPED:
      return "CUBEB_STATE_STOPPED";
    case CUBEB_STATE_DRAINED:
      return "CUBEB_STATE_DRAINED";
    case CUBEB_STATE_ERROR:
      return "CUBEB_STATE_ERROR";
    default:
      return "Undefined state";
  }
}

void print_log(const char* msg, ...) {
  va_list args;
  va_start(args, msg);
  vprintf(msg, args);
  va_end(args);
}

class cubeb_client final {
public:
  cubeb_client() {}
  ~cubeb_client() {}

  bool init(char const * backend_name = nullptr);
  bool init_stream();
  bool start_stream() const;
  bool stop_stream() const;
  bool destroy_stream() const;
  bool destroy();
  bool activate_log(bool active);

  long user_data_cb(cubeb_stream* stm, void* user, const void* input_buffer,
                    void* output_buffer, long nframes);

  void user_state_cb(cubeb_stream* stm, void* user, cubeb_state state);

  bool register_device_collection_changed(cubeb_device_type devtype);
  void device_collection_changed_callback(cubeb* context, void* user);

  cubeb_stream_params output_params = {};
  cubeb_stream_params input_params = {};

private:
  bool has_input() { return input_params.rate != 0; }
  bool has_output() { return output_params.rate != 0; }

  cubeb* context = nullptr;

  cubeb_stream* stream = nullptr;
  cubeb_devid output_device = nullptr;
  cubeb_devid input_device = nullptr;

  /* Accessed only from client and audio thread. */
  std::atomic<uint32_t> _rate = {0};
  std::atomic<uint32_t> _channels = {0};

  /* Accessed only from audio thread. */
  uint32_t _total_frames = 0;
};

bool cubeb_client::init(char const * backend_name) {
  int rv = cubeb_init(&context, "Cubeb Test Application", nullptr);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not init cubeb\n");
    return false;
  }
  return true;
}

static long user_data_cb_s(cubeb_stream* stm, void* user,
                           const void* input_buffer, void* output_buffer,
                           long nframes) {
  assert(user);
  return static_cast<cubeb_client*>(user)->user_data_cb(stm, user, input_buffer,
                                                        output_buffer, nframes);
}

static void user_state_cb_s(cubeb_stream* stm, void* user, cubeb_state state) {
  assert(user);
  return static_cast<cubeb_client*>(user)->user_state_cb(stm, user, state);
}

void device_collection_changed_callback_s(cubeb* context, void* user) {
  assert(user);
  return static_cast<cubeb_client*>(user)->device_collection_changed_callback(
      context, user);
}

bool cubeb_client::init_stream() {
  assert(has_input() || has_output());

  _rate = has_output() ? output_params.rate : input_params.rate;
  _channels = has_output() ? output_params.channels : input_params.channels;

  int rv =
      cubeb_stream_init(context, &stream, "Stream", input_device,
                        has_input() ? &input_params : nullptr, output_device,
                        has_output() ? &output_params : nullptr, 512,
                        user_data_cb_s, user_state_cb_s, this);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not open the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::start_stream() const {
  int rv = cubeb_stream_start(stream);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not start the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::stop_stream() const {
  int rv = cubeb_stream_stop(stream);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not stop the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::destroy_stream() const {
  cubeb_stream_destroy(stream);
  return true;
}

bool cubeb_client::destroy() {
  cubeb_destroy(context);
  return true;
}

bool cubeb_client::activate_log(bool active) {
  cubeb_log_level log_level = CUBEB_LOG_DISABLED;
  cubeb_log_callback log_callback = nullptr;
  if (active) {
    log_level = CUBEB_LOG_NORMAL;
    log_callback = print_log;
  }

  if (cubeb_set_log_callback(log_level, log_callback) != CUBEB_OK) {
    fprintf(stderr, "Set log callback failed\n");
    return false;
  }
  return true;
}

static void fill_with_sine_tone(float* buf, uint32_t num_of_frames,
                               uint32_t num_of_channels, uint32_t frame_rate,
                               uint32_t position) {
  for (uint32_t i = 0; i < num_of_frames; ++i) {
    for (uint32_t c = 0; c < num_of_channels; ++c) {
      buf[i * num_of_channels + c] =
          0.2 * sin(2 * M_PI * (i + position) * 350 / frame_rate);
      buf[i * num_of_channels + c] +=
          0.2 * sin(2 * M_PI * (i + position) * 440 / frame_rate);
    }
  }
}

long cubeb_client::user_data_cb(cubeb_stream* stm, void* user,
                                const void* input_buffer, void* output_buffer,
                                long nframes) {
  if (input_buffer && output_buffer) {
    const float* in = static_cast<const float*>(input_buffer);
    float* out = static_cast<float*>(output_buffer);
    memcpy(out, in, sizeof(float) * nframes * _channels);
  } else if (output_buffer && !input_buffer) {
    fill_with_sine_tone(static_cast<float*>(output_buffer), nframes, _channels,
                       _rate, _total_frames);
  }

  _total_frames += nframes;
  return nframes;
}

void cubeb_client::user_state_cb(cubeb_stream* stm, void* user,
                                 cubeb_state state) {
  fprintf(stderr, "state is %s\n", state_to_string(state));
}

bool cubeb_client::register_device_collection_changed(
    cubeb_device_type devtype) {
  int r = 0;
  r = cubeb_register_device_collection_changed(
      context, devtype, device_collection_changed_callback_s, this);
  if (r != CUBEB_OK) {
    return false;
  }
  return true;
}

void cubeb_client::device_collection_changed_callback(cubeb* context,
                                                      void* user) {
  fprintf(stderr, "device_collection_changed_callback\n");
}

enum play_mode {
  RECORD,
  PLAYBACK,
  DUPLEX,
  COLLECTION_CHANGE,
};

bool choose_action(const cubeb_client& cl, play_mode pm, char c) {
  while (c == 10 || c == 32) {
    // Consume "enter and "space"
    c = getchar();
  }

  if (c == 's') {
    if (pm == PLAYBACK || pm == RECORD || pm == DUPLEX) {
      cl.stop_stream();
      cl.destroy_stream();
    }
    return false;
  }

  fprintf(stderr, "Error: %c is not a valid entry\n", c);
  return true;
}

int main(int argc, char* argv[]) {
#ifdef _WIN32
  CoInitialize(nullptr);
#endif

  play_mode pm = DUPLEX;
  if (argc > 1) {
    if ('r' == argv[1][0]) {
      pm = RECORD;
    } else if ('p' == argv[1][0]) {
      pm = PLAYBACK;
    } else if ('d' == argv[1][0]) {
      pm = DUPLEX;
    } else if ('c' == argv[1][0]) {
      pm = COLLECTION_CHANGE;
    }
  }
  uint32_t rate = DEFAULT_RATE;
  if (argc > 2) {
    rate = strtoul(argv[2], NULL, 0);
  }

  bool res = false;
  cubeb_client cl;
  cl.activate_log(true);
  cl.init();

  if (pm == COLLECTION_CHANGE) {
    res = cl.register_device_collection_changed(static_cast<cubeb_device_type>(
        CUBEB_DEVICE_TYPE_INPUT & CUBEB_DEVICE_TYPE_INPUT));
    if (res) {
      fprintf(stderr,
              "register_device_collection_changed for input+output devices "
              "success\n");
    } else {
      fprintf(stderr, "register_device_collection_changed failed\n");
    }
  } else {
    if (pm == PLAYBACK || pm == DUPLEX) {
      cl.output_params = {CUBEB_SAMPLE_FLOAT32NE, rate, DEFAULT_CHANNELS,
                          CUBEB_LAYOUT_UNDEFINED, CUBEB_STREAM_PREF_NONE};
    }
    if (pm == RECORD || pm == DUPLEX) {
      cl.input_params = {CUBEB_SAMPLE_FLOAT32NE, rate, DEFAULT_CHANNELS,
                         CUBEB_LAYOUT_UNDEFINED, CUBEB_STREAM_PREF_NONE};
    }
    res = cl.init_stream();
    if (!res) {
      return 1;
    }

    cl.start_stream();
  }

  // User input
  do {
    fprintf(stderr, "press `s` to stop the stream (if any) and abort\n");
  } while (choose_action(cl, pm, getchar()));

  cl.destroy();

  return 0;
}
<commit_msg>cubeb-test: improvements for testing collection changed callback (#489)<commit_after>#include "cubeb/cubeb.h"
#include <atomic>
#include <cassert>
#include <cmath>
#include <cstdarg>
#include <cstring>
#include <iostream>
#ifdef _WIN32
#include <objbase.h> // Used by CoInitialize()
#endif

#ifndef M_PI
#define M_PI 3.14159263
#endif

// Default values if none specified
#define DEFAULT_RATE 44100
#define DEFAULT_CHANNELS 2

static const char* state_to_string(cubeb_state state) {
  switch (state) {
    case CUBEB_STATE_STARTED:
      return "CUBEB_STATE_STARTED";
    case CUBEB_STATE_STOPPED:
      return "CUBEB_STATE_STOPPED";
    case CUBEB_STATE_DRAINED:
      return "CUBEB_STATE_DRAINED";
    case CUBEB_STATE_ERROR:
      return "CUBEB_STATE_ERROR";
    default:
      return "Undefined state";
  }
}

void print_log(const char* msg, ...) {
  va_list args;
  va_start(args, msg);
  vprintf(msg, args);
  va_end(args);
}

class cubeb_client final {
public:
  cubeb_client() {}
  ~cubeb_client() {}

  bool init(char const * backend_name = nullptr);
  bool init_stream();
  bool start_stream() const;
  bool stop_stream() const;
  bool destroy_stream() const;
  bool destroy();
  bool activate_log(bool active);

  long user_data_cb(cubeb_stream* stm, void* user, const void* input_buffer,
                    void* output_buffer, long nframes);

  void user_state_cb(cubeb_stream* stm, void* user, cubeb_state state);

  bool register_device_collection_changed(cubeb_device_type devtype) const;
  bool unregister_device_collection_changed(cubeb_device_type devtype) const;

  cubeb_stream_params output_params = {};
  cubeb_stream_params input_params = {};

private:
  bool has_input() { return input_params.rate != 0; }
  bool has_output() { return output_params.rate != 0; }

  cubeb* context = nullptr;

  cubeb_stream* stream = nullptr;
  cubeb_devid output_device = nullptr;
  cubeb_devid input_device = nullptr;

  /* Accessed only from client and audio thread. */
  std::atomic<uint32_t> _rate = {0};
  std::atomic<uint32_t> _channels = {0};

  /* Accessed only from audio thread. */
  uint32_t _total_frames = 0;
};

bool cubeb_client::init(char const * backend_name) {
  int rv = cubeb_init(&context, "Cubeb Test Application", nullptr);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not init cubeb\n");
    return false;
  }
  return true;
}

static long user_data_cb_s(cubeb_stream* stm, void* user,
                           const void* input_buffer, void* output_buffer,
                           long nframes) {
  assert(user);
  return static_cast<cubeb_client*>(user)->user_data_cb(stm, user, input_buffer,
                                                        output_buffer, nframes);
}

static void user_state_cb_s(cubeb_stream* stm, void* user, cubeb_state state) {
  assert(user);
  return static_cast<cubeb_client*>(user)->user_state_cb(stm, user, state);
}

void input_device_changed_callback_s(cubeb* context, void* user) {
  fprintf(stderr, "input_device_changed_callback_s\n");
}

void output_device_changed_callback_s(cubeb* context, void* user) {
  fprintf(stderr, "output_device_changed_callback_s\n");
}

void io_device_changed_callback_s(cubeb* context, void* user) {
  fprintf(stderr, "io_device_changed_callback\n");
}

bool cubeb_client::init_stream() {
  assert(has_input() || has_output());

  _rate = has_output() ? output_params.rate : input_params.rate;
  _channels = has_output() ? output_params.channels : input_params.channels;

  int rv =
      cubeb_stream_init(context, &stream, "Stream", input_device,
                        has_input() ? &input_params : nullptr, output_device,
                        has_output() ? &output_params : nullptr, 512,
                        user_data_cb_s, user_state_cb_s, this);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not open the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::start_stream() const {
  int rv = cubeb_stream_start(stream);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not start the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::stop_stream() const {
  int rv = cubeb_stream_stop(stream);
  if (rv != CUBEB_OK) {
    fprintf(stderr, "Could not stop the stream\n");
    return false;
  }
  return true;
}

bool cubeb_client::destroy_stream() const {
  cubeb_stream_destroy(stream);
  return true;
}

bool cubeb_client::destroy() {
  cubeb_destroy(context);
  return true;
}

bool cubeb_client::activate_log(bool active) {
  cubeb_log_level log_level = CUBEB_LOG_DISABLED;
  cubeb_log_callback log_callback = nullptr;
  if (active) {
    log_level = CUBEB_LOG_NORMAL;
    log_callback = print_log;
  }

  if (cubeb_set_log_callback(log_level, log_callback) != CUBEB_OK) {
    fprintf(stderr, "Set log callback failed\n");
    return false;
  }
  return true;
}

static void fill_with_sine_tone(float* buf, uint32_t num_of_frames,
                               uint32_t num_of_channels, uint32_t frame_rate,
                               uint32_t position) {
  for (uint32_t i = 0; i < num_of_frames; ++i) {
    for (uint32_t c = 0; c < num_of_channels; ++c) {
      buf[i * num_of_channels + c] =
          0.2 * sin(2 * M_PI * (i + position) * 350 / frame_rate);
      buf[i * num_of_channels + c] +=
          0.2 * sin(2 * M_PI * (i + position) * 440 / frame_rate);
    }
  }
}

long cubeb_client::user_data_cb(cubeb_stream* stm, void* user,
                                const void* input_buffer, void* output_buffer,
                                long nframes) {
  if (input_buffer && output_buffer) {
    const float* in = static_cast<const float*>(input_buffer);
    float* out = static_cast<float*>(output_buffer);
    memcpy(out, in, sizeof(float) * nframes * _channels);
  } else if (output_buffer && !input_buffer) {
    fill_with_sine_tone(static_cast<float*>(output_buffer), nframes, _channels,
                       _rate, _total_frames);
  }

  _total_frames += nframes;
  return nframes;
}

void cubeb_client::user_state_cb(cubeb_stream* stm, void* user,
                                 cubeb_state state) {
  fprintf(stderr, "state is %s\n", state_to_string(state));
}

bool cubeb_client::register_device_collection_changed(
    cubeb_device_type devtype) const {
  cubeb_device_collection_changed_callback callback = nullptr;
  if (devtype == static_cast<cubeb_device_type>(CUBEB_DEVICE_TYPE_INPUT |
                                                CUBEB_DEVICE_TYPE_OUTPUT)) {
    callback = io_device_changed_callback_s;
  } else if (devtype & CUBEB_DEVICE_TYPE_OUTPUT) {
    callback = output_device_changed_callback_s;
  } else if (devtype & CUBEB_DEVICE_TYPE_INPUT) {
    callback = input_device_changed_callback_s;
  }
  int r = cubeb_register_device_collection_changed(
            context, devtype, callback, nullptr);
  if (r != CUBEB_OK) {
    return false;
  }
  return true;
}

bool cubeb_client::unregister_device_collection_changed(
    cubeb_device_type devtype) const {
  int r = cubeb_register_device_collection_changed(
            context, devtype, nullptr, nullptr);
  if (r != CUBEB_OK) {
    return false;
  }
  return true;
}

enum play_mode {
  RECORD,
  PLAYBACK,
  DUPLEX,
  COLLECTION_CHANGE,
};

struct operation_data {
  play_mode pm;
  uint32_t rate;
  cubeb_device_type collection_device_type;
};

void print_help() {
  const char * msg =
    "p: start a initialized stream\n"
    "s: stop a started stream\n"
    "i: change device type to input\n"
    "o: change device type to output\n"
    "a: change device type to input and output\n"
    "k: change device type to unknown\n"
    "r: register device collection changed callback for the current device type\n"
    "u: unregister device collection changed callback for the current device type\n"
    "q: quit\n"
    "h: print this message\n";
  fprintf(stderr, "%s\n", msg);
}

bool choose_action(const cubeb_client& cl, operation_data * op, char c) {
  while (c == 10 || c == 32) {
    // Consume "enter and "space"
    c = getchar();
  }

  if (c == 'q') {
    if (op->pm == PLAYBACK || op->pm == RECORD || op->pm == DUPLEX) {
      bool res = cl.stop_stream();
      if (!res) {
        fprintf(stderr, "stop_stream failed\n");
      }
      res = cl.destroy_stream();
      if (!res) {
        fprintf(stderr, "destroy_stream failed\n");
      }
    } else if (op->pm == COLLECTION_CHANGE) {
      bool res = cl.unregister_device_collection_changed(op->collection_device_type);
      if (!res) {
        fprintf(stderr, "unregister_device_collection_changed failed\n");
      }
    }
    return false; // exit the loop
  } else if (c == 'h') {
    print_help();
  } else if (c == 'p') {
    bool res = cl.start_stream();
    if (res) {
      fprintf(stderr, "start_stream succeed\n");
    } else {
      fprintf(stderr, "start_stream failed\n");
    }
  } else if (c == 's') {
    bool res = cl.stop_stream();
    if (res) {
      fprintf(stderr, "stop_stream succeed\n");
    } else {
      fprintf(stderr, "stop_stream failed\n");
    }
  } else if (c == 'i') {
    op->collection_device_type = CUBEB_DEVICE_TYPE_INPUT;
    fprintf(stderr, "collection device type changed to INPUT\n");
  } else if (c == 'o') {
    op->collection_device_type = CUBEB_DEVICE_TYPE_OUTPUT;
    fprintf(stderr, "collection device type changed to OUTPUT\n");
  } else if (c == 'a') {
    op->collection_device_type = static_cast<cubeb_device_type>(CUBEB_DEVICE_TYPE_INPUT | CUBEB_DEVICE_TYPE_OUTPUT);
    fprintf(stderr, "collection device type changed to INPUT | OUTPUT\n");
  } else if (c == 'k') {
    op->collection_device_type = CUBEB_DEVICE_TYPE_UNKNOWN;
    fprintf(stderr, "collection device type changed to UNKNOWN\n");
  } else if (c == 'r') {
    bool res = cl.register_device_collection_changed(op->collection_device_type);
    if (res) {
      fprintf(stderr, "register_device_collection_changed succeed\n");
    } else {
      fprintf(stderr, "register_device_collection_changed failed\n");
    }
  } else if (c == 'u') {
    bool res = cl.unregister_device_collection_changed(op->collection_device_type);
    if (res) {
      fprintf(stderr, "unregister_device_collection_changed succeed\n");
    } else {
      fprintf(stderr, "unregister_device_collection_changed failed\n");
    }
  } else {
    fprintf(stderr, "Error: %c is not a valid entry\n", c);
  }

  return true; // Loop up
}

int main(int argc, char* argv[]) {
#ifdef _WIN32
  CoInitialize(nullptr);
#endif

  operation_data op;
  op.pm = PLAYBACK;
  if (argc > 1) {
    if ('r' == argv[1][0]) {
      op.pm = RECORD;
    } else if ('p' == argv[1][0]) {
      op.pm = PLAYBACK;
    } else if ('d' == argv[1][0]) {
      op.pm = DUPLEX;
    } else if ('c' == argv[1][0]) {
      op.pm = COLLECTION_CHANGE;
    }
  }
  op.rate = DEFAULT_RATE;
  if (argc > 2) {
    op.rate = strtoul(argv[2], NULL, 0);
  }

  bool res = false;
  cubeb_client cl;
  cl.activate_log(true);
  cl.init();

  op.collection_device_type = CUBEB_DEVICE_TYPE_UNKNOWN;
  fprintf(stderr, "collection device type is UNKNOWN\n");
  if (op.pm == COLLECTION_CHANGE) {
    op.collection_device_type = CUBEB_DEVICE_TYPE_OUTPUT;
    fprintf(stderr, "collection device type changed to OUTPUT\n");
    res = cl.register_device_collection_changed(op.collection_device_type);
    if (res) {
      fprintf(stderr, "register_device_collection_changed succeed\n");
    } else {
      fprintf(stderr, "register_device_collection_changed failed\n");
    }
  } else {
    if (op.pm == PLAYBACK || op.pm == DUPLEX) {
      cl.output_params = {CUBEB_SAMPLE_FLOAT32NE, op.rate, DEFAULT_CHANNELS,
                          CUBEB_LAYOUT_UNDEFINED, CUBEB_STREAM_PREF_NONE};
    }
    if (op.pm == RECORD || op.pm == DUPLEX) {
      cl.input_params = {CUBEB_SAMPLE_FLOAT32NE, op.rate, DEFAULT_CHANNELS,
                         CUBEB_LAYOUT_UNDEFINED, CUBEB_STREAM_PREF_NONE};
    }
    res = cl.init_stream();
    if (!res) {
      fprintf(stderr, "stream_init failed\n");
      return -1;
    }
    fprintf(stderr, "stream_init succeed\n");

    res = cl.start_stream();
    if (res) {
      fprintf(stderr, "stream_start succeed\n");
    } else {
      fprintf(stderr, "stream_init failed\n");
    }
  }

  // User input
  do {
    fprintf(stderr, "press `q` to abort or `h` for help\n");
  } while (choose_action(cl, &op, getchar()));

  cl.destroy();

  return 0;
}
<|endoftext|>


<commit_before>/// @brief provides fixed-point sum info

#ifndef INC_SUM_INFO_HPP_
#define INC_SUM_INFO_HPP_

#include <boost/integer.hpp>
#include <boost/static_assert.hpp>
#include <boost/concept_check.hpp>

#include <boost/mpl/if.hpp>
#include <boost/mpl/eval_if.hpp>

#include <limits>
#include <boost/cstdint.hpp>

namespace utils {
    namespace {
        using boost::mpl::if_;
        using boost::mpl::eval_if;
    }

    template<typename storage_type, size_t total, size_t fractionals>
    class number;

    /// @brief tool for type inference of the summ result with fixed-point and
    /// floating-point numbers
    template<typename value_type>
    class sum_info
    {
    public:
        typedef value_type sum_type;
        typedef value_type sum_value_type;

        struct sign_info
        {
            enum { value = true };
        };

        struct closing_info
        {
            enum { value = true };
        };

        struct value_type_info
        {
            struct op
            {
                typedef value_type type;
            };

            struct cl
            {
                typedef value_type type;
            };
        };
    };

    /// @brief in case of fixed-point numbers
    template<typename T, size_t n, size_t f>
    class sum_info<number<T, n, f> >
    {
        typedef number<T, n, f> operand_type;

    public:
        ///< is the type of summ a signed type?
        struct is_signed
        {
            enum { value = std::numeric_limits<T>::is_signed };
        };

        ///< is type of the summ closed under arithmetic operations?
        struct is_closed
        {
            typedef typename if_<is_signed, boost::intmax_t, boost::uintmax_t>::type
                max_type;

            // total bits and std::numeric_limits do not count sign bit
            enum { value = !(n + 1 <= std::numeric_limits<max_type>::digits) };
        };

        // brief: 1. if one do not have integral type of enough bits count then
        //           it has to interpret summation as a closed operation;
        //        2. boost::int_t template parameter takes into account a sign bit.
        ///< integral value type below fixed-point number as a summ result type
        struct value_type_info
        {
            ///< in case if summ type is not closed under arithmetic operations
            struct op
            {
                typedef typename if_<is_signed, typename boost::int_t<n + 2u>::least,
                    typename boost::uint_t<n + 1u>::least>::type type;
            };

            /// @brief in case if sum type is closed under arithmetic operations
            struct cl
            {
                typedef T type;
            };
        };

        ///< integral value type that is below of fixed-point result type of the summ
        typedef typename eval_if<is_closed, typename value_type_info::cl,
            typename value_type_info::op>::type sum_value_type;

        ///< fixed-point type for summ result
        typedef typename if_<is_closed, operand_type, number<sum_value_type, n + 1, f> >::type
            sum_type;
    };
}

#endif
<commit_msg>sum_info.hpp: keep code uniform -> 1 to 1u<commit_after>/// @brief provides fixed-point sum info

#ifndef INC_SUM_INFO_HPP_
#define INC_SUM_INFO_HPP_

#include <boost/integer.hpp>
#include <boost/static_assert.hpp>
#include <boost/concept_check.hpp>

#include <boost/mpl/if.hpp>
#include <boost/mpl/eval_if.hpp>

#include <limits>
#include <boost/cstdint.hpp>

namespace utils {
    namespace {
        using boost::mpl::if_;
        using boost::mpl::eval_if;
    }

    template<typename storage_type, size_t total, size_t fractionals>
    class number;

    /// @brief tool for type inference of the summ result with fixed-point and
    /// floating-point numbers
    template<typename value_type>
    class sum_info
    {
    public:
        typedef value_type sum_type;
        typedef value_type sum_value_type;

        struct sign_info
        {
            enum { value = true };
        };

        struct closing_info
        {
            enum { value = true };
        };

        struct value_type_info
        {
            struct op
            {
                typedef value_type type;
            };

            struct cl
            {
                typedef value_type type;
            };
        };
    };

    /// @brief in case of fixed-point numbers
    template<typename T, size_t n, size_t f>
    class sum_info<number<T, n, f> >
    {
        typedef number<T, n, f> operand_type;

    public:
        ///< is the type of summ a signed type?
        struct is_signed
        {
            enum { value = std::numeric_limits<T>::is_signed };
        };

        ///< is type of the summ closed under arithmetic operations?
        struct is_closed
        {
            typedef typename if_<is_signed, boost::intmax_t, boost::uintmax_t>::type
                max_type;

            // total bits and std::numeric_limits do not count sign bit
            enum { value = !(n + 1u <= std::numeric_limits<max_type>::digits) };
        };

        // brief: 1. if one do not have integral type of enough bits count then
        //           it has to interpret summation as a closed operation;
        //        2. boost::int_t template parameter takes into account a sign bit.
        ///< integral value type below fixed-point number as a summ result type
        struct value_type_info
        {
            ///< in case if summ type is not closed under arithmetic operations
            struct op
            {
                typedef typename if_<is_signed, typename boost::int_t<n + 2u>::least,
                    typename boost::uint_t<n + 1u>::least>::type type;
            };

            /// @brief in case if sum type is closed under arithmetic operations
            struct cl
            {
                typedef T type;
            };
        };

        ///< integral value type that is below of fixed-point result type of the summ
        typedef typename eval_if<is_closed, typename value_type_info::cl,
            typename value_type_info::op>::type sum_value_type;

        ///< fixed-point type for summ result
        typedef typename if_<is_closed, operand_type, number<sum_value_type, n + 1u, f> >::type
            sum_type;
    };
}

#endif
<|endoftext|>


<commit_before>#pragma once
#include "CrestCacheEntry.hpp"
#include "CrestConnectionPool.hpp"
#include <string>
#include <unordered_map>
#include <mutex>

class CrestCache
{
public:
    struct CacheLookupResults
    {
        CrestCacheEntry *entry;
        /**Lock on entry->mutex*/
        std::unique_lock<std::mutex> lock;
    };
    struct CacheLookupFutureResults
    {
        CrestCacheEntry *entry;
        CrestCacheEntry::Status status;
        std::vector<uint8_t> data;
        void wait()
        {
            std::unique_lock<std::mutex> lock(entry->mutex);
            entry->wait(lock);
            status = entry->status;
            data = entry->data;
        }
    };

    CrestCache();
    ~CrestCache();
    void stop();

    CrestCacheEntry *get(const std::string &path);
    /**Get CREST data.
     * @param path The path to get from CREST.
     */
    CacheLookupFutureResults get_future(const std::string &path);
    CacheLookupResults get_now(const std::string &path);

    /**Used by CrestConnectionPool to use up spare bandwidth by getting requests to cache in
     * advance.
     */
    CrestHttpRequest *get_preload_request();
private:
    typedef std::unique_lock<std::mutex> unique_lock;
    typedef std::unordered_map<std::string, CrestCacheEntry> Cache;
    static const size_t MAX_CACHE_SIZE = 1024*1024*20//20MB

    /**Lock for the cache map.
     * @warning Never attempt to lock this if already holding any entry lock. Always lock this
     * before locking an entry. Failure to do so may result in dead lock.
     */
    std::mutex cache_mutex;
    size_t cache_size;
    Cache cache;
    /**CrestConnectionPool for updating cache entries.*/
    std::vector<CrestCacheEntry*> preload_entries;
    size_t preload_request_next;
    CrestConnectionPool crest_connection_pool;

    CrestCacheEntry &get_entry(const std::string &path);
    CrestCacheEntry *get_locked(const std::string &path, std::unique_lock<std::mutex> &lock);
    void update_entry(CrestCacheEntry &entry);
    void update_entry_completion(CrestCacheEntry *entry, CrestHttpRequest *request);
    /**With the cache already locked, see if any data must be purged to remaining within
     * cache size limits.
     */
    void check_cache_purge();
};
<commit_msg>Corrected syntax error in cache limit<commit_after>#pragma once
#include "CrestCacheEntry.hpp"
#include "CrestConnectionPool.hpp"
#include <string>
#include <unordered_map>
#include <mutex>

class CrestCache
{
public:
    struct CacheLookupResults
    {
        CrestCacheEntry *entry;
        /**Lock on entry->mutex*/
        std::unique_lock<std::mutex> lock;
    };
    struct CacheLookupFutureResults
    {
        CrestCacheEntry *entry;
        CrestCacheEntry::Status status;
        std::vector<uint8_t> data;
        void wait()
        {
            std::unique_lock<std::mutex> lock(entry->mutex);
            entry->wait(lock);
            status = entry->status;
            data = entry->data;
        }
    };

    CrestCache();
    ~CrestCache();
    void stop();

    CrestCacheEntry *get(const std::string &path);
    /**Get CREST data.
     * @param path The path to get from CREST.
     */
    CacheLookupFutureResults get_future(const std::string &path);
    CacheLookupResults get_now(const std::string &path);

    /**Used by CrestConnectionPool to use up spare bandwidth by getting requests to cache in
     * advance.
     */
    CrestHttpRequest *get_preload_request();
private:
    typedef std::unique_lock<std::mutex> unique_lock;
    typedef std::unordered_map<std::string, CrestCacheEntry> Cache;
    static const size_t MAX_CACHE_SIZE = 1024*1024*20;//20MB

    /**Lock for the cache map.
     * @warning Never attempt to lock this if already holding any entry lock. Always lock this
     * before locking an entry. Failure to do so may result in dead lock.
     */
    std::mutex cache_mutex;
    size_t cache_size;
    Cache cache;
    /**CrestConnectionPool for updating cache entries.*/
    std::vector<CrestCacheEntry*> preload_entries;
    size_t preload_request_next;
    CrestConnectionPool crest_connection_pool;

    CrestCacheEntry &get_entry(const std::string &path);
    CrestCacheEntry *get_locked(const std::string &path, std::unique_lock<std::mutex> &lock);
    void update_entry(CrestCacheEntry &entry);
    void update_entry_completion(CrestCacheEntry *entry, CrestHttpRequest *request);
    /**With the cache already locked, see if any data must be purged to remaining within
     * cache size limits.
     */
    void check_cache_purge();
};
<|endoftext|>