<commit_before>#include "switch.h"
#include "Arduino.h"

#define LIMIT 3

Switch::Switch(int relayPin, int lightButtonPin, int modifyButtonPin, int ticker, int delay)
{
    relay = relayPin;
    lightButton = lightButtonPin;
    modifyButton = modifyButtonPin;

    pinMode(relay, OUTPUT);
    pinMode(lightButton, INPUT_PULLUP);
    pinMode(modifyButton, INPUT_PULLUP);

    digitalWrite(relay, LOW);

    lightBtnPushed = 0;
    modifyBtnPushed = 0;
    isOn = false;
    tickerDefault = ticker;
    delayTime = delay;
}

void Switch::handle()
{
    readButtons();

    if(lightBtnPushed > LIMIT)                  //if interaction...
    {
        if(isOn)
        {
            digitalWrite(relay, LOW);           //turn light off
            isOn = false;

            waitForRelease(lightBtnPushed);     //wait while button is still hold
        }
        else
        {
            digitalWrite(relay, HIGH);          //turn light on
            isOn = true;
            cycleTicker = tickerDefault;
            cycleCount = 1;

            checkModify();
        }
    }
    else                                        //if no interaction...
    {
        if(isOn)
        {
            cycleTicker--;
            if(cycleTicker <= 0)
            {
                cycleCount--;
                if(cycleCount <= 0)
                {
                    digitalWrite(relay, LOW);   //turn light off
                    isOn = false;

                    cycleCount = 0;
                    cycleTicker = 0;
                }
                else
                {
                    cycleTicker = tickerDefault;
                }
            }
        }
    }
}

void Switch::readButtons()
{
     if(digitalRead(lightButton) == LOW)
     {
         lightBtnPushed++;
         if(lightBtnPushed > 10000)             //prevent overflow
         {
             lightBtnPushed = 10000;
         }
     }
     else
     {
         lightBtnPushed = 0;
     }

     if(digitalRead(modifyButton) == LOW)
     {
         modifyBtnPushed++;
         if(modifyBtnPushed > 10000)            //prevent overflow
         {
             modifyBtnPushed = 10000;
         }
     }
     else
     {
         modifyBtnPushed = 0;
     }
}

void Switch::waitForRelease(int &button)        //wait while the buttin in the reference is still pressed
{
    while(button > LIMIT)
    {
        readButtons();
        delay(delayTime);
    }
}

void Switch::checkModify()                      //doubles the cycleCount every time the second button is pressed
{
    while(lightBtnPushed > LIMIT)
    {
        readButtons();
        if(modifyBtnPushed > LIMIT)
        {
            cycleCount = cycleCount * 2;
            waitForRelease(modifyBtnPushed);    //wait to release modify button
        }
        delay(10);
    }
}

<commit_msg>changed polarity of outputs because of reasons<commit_after>#include "switch.h"
#include "Arduino.h"

#define LIMIT 3

Switch::Switch(int relayPin, int lightButtonPin, int modifyButtonPin, int ticker, int delay)
{
    relay = relayPin;
    lightButton = lightButtonPin;
    modifyButton = modifyButtonPin;

    pinMode(relay, OUTPUT);
    pinMode(lightButton, INPUT_PULLUP);
    pinMode(modifyButton, INPUT_PULLUP);

    digitalWrite(relay, HIGH);                  //HIGH is off

    lightBtnPushed = 0;
    modifyBtnPushed = 0;
    isOn = false;
    tickerDefault = ticker;
    delayTime = delay;
}

void Switch::handle()
{
    readButtons();

    if(lightBtnPushed > LIMIT)                  //if interaction...
    {
        if(isOn)
        {
            digitalWrite(relay, HIGH);          //turn light off
            isOn = false;

            waitForRelease(lightBtnPushed);     //wait while button is still hold
        }
        else
        {
            digitalWrite(relay, LOW);           //turn light on
            isOn = true;
            cycleTicker = tickerDefault;
            cycleCount = 1;

            checkModify();
        }
    }
    else                                        //if no interaction...
    {
        if(isOn)
        {
            cycleTicker--;
            if(cycleTicker <= 0)
            {
                cycleCount--;
                if(cycleCount <= 0)
                {
                    digitalWrite(relay, HIGH);  //turn light off
                    isOn = false;

                    cycleCount = 0;
                    cycleTicker = 0;
                }
                else
                {
                    cycleTicker = tickerDefault;
                }
            }
        }
    }
}

void Switch::readButtons()
{
     if(digitalRead(lightButton) == LOW)
     {
         lightBtnPushed++;
         if(lightBtnPushed > 10000)             //prevent overflow
         {
             lightBtnPushed = 10000;
         }
     }
     else
     {
         lightBtnPushed = 0;
     }

     if(digitalRead(modifyButton) == LOW)
     {
         modifyBtnPushed++;
         if(modifyBtnPushed > 10000)            //prevent overflow
         {
             modifyBtnPushed = 10000;
         }
     }
     else
     {
         modifyBtnPushed = 0;
     }
}

void Switch::waitForRelease(int &button)        //wait while the buttin in the reference is still pressed
{
    while(button > LIMIT)
    {
        readButtons();
        delay(delayTime);
    }
}

void Switch::checkModify()                      //doubles the cycleCount every time the second button is pressed
{
    while(lightBtnPushed > LIMIT)
    {
        readButtons();
        if(modifyBtnPushed > LIMIT)
        {
            cycleCount = cycleCount * 2;
            waitForRelease(modifyBtnPushed);    //wait to release modify button
        }
        delay(10);
    }
}

<|endoftext|>


<commit_before>/*
Copyright 2014-2015 Adam Grandquist

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */
/**
 * @author Adam Grandquist
 * @copyright Apache
 */

#ifndef REQL_CPP_CONNECTION_HPP_
#define REQL_CPP_CONNECTION_HPP_

#include "./cpp/error.hpp"

#include <cstdint>
#include <string>

namespace ReQL {

class Connection {
public:
  Connection();
  explicit Connection(const std::string &host);
  Connection(const std::string &host, const std::uint16_t &port);
  Connection(const std::string &host, const std::uint16_t &port, const std::string &key);

  Connection(Connection &&other);

  Connection &operator=(Connection &&other);

  bool isOpen() const;

  void close();

  void reconnect();

private:

  friend class Query;
  std::shared_ptr<_ReQL::Conn_t<ReQLError, ReQLError, Result, ReQLError>> p_conn;
};

}  // namespace ReQL

#endif  // REQL_CPP_CONNECTION_HPP_
<commit_msg>[cpp] Add include for shared_ptr.<commit_after>/*
Copyright 2014-2015 Adam Grandquist

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */
/**
 * @author Adam Grandquist
 * @copyright Apache
 */

#ifndef REQL_CPP_CONNECTION_HPP_
#define REQL_CPP_CONNECTION_HPP_

#include "./cpp/error.hpp"

#include <cstdint>
#include <memory>
#include <string>

namespace ReQL {

class Connection {
public:
  Connection();
  explicit Connection(const std::string &host);
  Connection(const std::string &host, const std::uint16_t &port);
  Connection(const std::string &host, const std::uint16_t &port, const std::string &key);

  Connection(Connection &&other);

  Connection &operator=(Connection &&other);

  bool isOpen() const;

  void close();

  void reconnect();

private:

  friend class Query;
  std::shared_ptr<_ReQL::Conn_t<ReQLError, ReQLError, Result, ReQLError>> p_conn;
};

}  // namespace ReQL

#endif  // REQL_CPP_CONNECTION_HPP_
<|endoftext|>


<commit_before>/**
 * This file defines alternative machinery to declare named template
 * parameters because of a bug in Boost.Parameter that might take a while
 * before its fixed.
 *
 * @see https://svn.boost.org/trac/boost/ticket/2793
 */

#ifndef D2_SANDBOX_PARAMETER_HPP
#define D2_SANDBOX_PARAMETER_HPP

#include <boost/parameter.hpp>

namespace d2 {
namespace sandbox {

namespace parameter_detail {
    template <typename T>
    struct template_parameter_wrapper { };

    template <typename T>
    struct unwrap_template_parameter {
        typedef T type;
    };

    template <typename T>
    struct unwrap_template_parameter<template_parameter_wrapper<T> > {
        typedef T type;
    };
} // end namespace parameter_detail

template <typename Parameters, typename Keyword,
          typename Default = boost::parameter::void_>
struct parameter_value_type
    : parameter_detail::unwrap_template_parameter<
        typename boost::parameter::value_type<
            Parameters, Keyword, Default
        >::type
    >
{ };

template <typename Tag, typename T>
struct parameter_template_keyword
    : boost::parameter::template_keyword<
        Tag, parameter_detail::template_parameter_wrapper<T>
    >
{ };

#define D2_PARAMETER_TEMPLATE_KEYWORD(keyword)                              \
    namespace tag { struct keyword; }                                       \
    template <typename T>                                                   \
    struct keyword                                                          \
        : ::d2::sandbox::parameter_template_keyword<                        \
            tag::keyword, T                                                 \
        >                                                                   \
    { };                                                                    \
/**/

} // end namespace sandbox
} // end namespace d2

#endif // !D2_SANDBOX_PARAMETER_HPP
<commit_msg>Fix typo.<commit_after>/**
 * This file defines alternative machinery to declare named template
 * parameters because of a bug in Boost.Parameter that might take a while
 * before it is fixed.
 *
 * @see https://svn.boost.org/trac/boost/ticket/2793
 */

#ifndef D2_SANDBOX_PARAMETER_HPP
#define D2_SANDBOX_PARAMETER_HPP

#include <boost/parameter.hpp>

namespace d2 {
namespace sandbox {

namespace parameter_detail {
    template <typename T>
    struct template_parameter_wrapper { };

    template <typename T>
    struct unwrap_template_parameter {
        typedef T type;
    };

    template <typename T>
    struct unwrap_template_parameter<template_parameter_wrapper<T> > {
        typedef T type;
    };
} // end namespace parameter_detail

template <typename Parameters, typename Keyword,
          typename Default = boost::parameter::void_>
struct parameter_value_type
    : parameter_detail::unwrap_template_parameter<
        typename boost::parameter::value_type<
            Parameters, Keyword, Default
        >::type
    >
{ };

template <typename Tag, typename T>
struct parameter_template_keyword
    : boost::parameter::template_keyword<
        Tag, parameter_detail::template_parameter_wrapper<T>
    >
{ };

#define D2_PARAMETER_TEMPLATE_KEYWORD(keyword)                              \
    namespace tag { struct keyword; }                                       \
    template <typename T>                                                   \
    struct keyword                                                          \
        : ::d2::sandbox::parameter_template_keyword<                        \
            tag::keyword, T                                                 \
        >                                                                   \
    { };                                                                    \
/**/

} // end namespace sandbox
} // end namespace d2

#endif // !D2_SANDBOX_PARAMETER_HPP
<|endoftext|>


<commit_before>#include "sensor.h"
#include <DHT.h>  // DHT sensor library
#include <Arduino.h>

// data pin definition for sensor
#define DHT_DATA       7
#define DHT_TYPE       DHT22
#define DHT_FAHRENHEIT false

namespace Sensor {
  DHT dht = DHT(DHT_DATA, DHT_TYPE);

  void setup() {
    // initalize the sensor
    dht.begin();
  }

  int readTemp() {
    return dht.readTemperature(DHT_FAHRENHEIT);
  }
}
<commit_msg>Add humidity reading<commit_after>#include "sensor.h"
#include <DHT.h>  // DHT sensor library
#include <Arduino.h>

// data pin definition for sensor
#define DHT_DATA       7
#define DHT_TYPE       DHT22
#define DHT_FAHRENHEIT false

namespace Sensor {
  DHT dht = DHT(DHT_DATA, DHT_TYPE);

  void setup() {
    // initalize the sensor
    dht.begin();
  }

  int readTemp() {
    return dht.readTemperature(DHT_FAHRENHEIT);
  }

  int readHum() {
    return dht.readHumidity();
  }
}
<|endoftext|>


<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include "fnord/stdtypes.h"
#include "fnord/application.h"
#include "fnord/cli/flagparser.h"
#include "fnord/cli/CLI.h"
#include "fnord/csv/CSVInputStream.h"
#include "fnord/io/file.h"
#include "fnord/inspect.h"
#include "fnord/human.h"
#include "cstable/LEB128ColumnWriter.h"
#include "cstable/StringColumnWriter.h"
#include "cstable/DoubleColumnWriter.h"
#include "cstable/BooleanColumnWriter.h"
#include "cstable/CSTableWriter.h"

using namespace fnord;

void cmd_from_csv(const cli::FlagParser& flags) {
  auto csv = CSVInputStream::openFile(
      flags.getString("input_file"),
      '\t',
      '\n');

  Vector<String> columns;
  csv->readNextRow(&columns);

  HashMap<String, HumanDataType> column_types;
  for (const auto& hdr : columns) {
    column_types[hdr] = HumanDataType::UNKNOWN;
  }

  Vector<String> row;
  while (csv->readNextRow(&row)) {
    for (size_t i = 0; i < row.size() && i < columns.size(); ++i) {
      auto& ctype = column_types[columns[i]];
      ctype = Human::detectDataTypeSeries(row[i], ctype);
    }
  }

  HashMap<String, RefPtr<cstable::ColumnWriter>> column_writers;
  for (const auto& col : column_types) {
    switch (col.second) {

        case HumanDataType::UNSIGNED_INTEGER:
          column_writers.emplace(
              col.first,
              new cstable::LEB128ColumnWriter(0, 0));
          break;

        case HumanDataType::UNSIGNED_INTEGER_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::LEB128ColumnWriter(0, 1));
          break;

        case HumanDataType::SIGNED_INTEGER:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 0)); // FIXME
          break;

        case HumanDataType::SIGNED_INTEGER_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 1)); // FIXME
          break;

        case HumanDataType::FLOAT:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 0));
          break;

        case HumanDataType::FLOAT_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 1));
          break;

        case HumanDataType::BOOLEAN:
          column_writers.emplace(
              col.first,
              new cstable::BooleanColumnWriter(0, 0));
          break;

        case HumanDataType::BOOLEAN_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::BooleanColumnWriter(0, 1));
          break;

        case HumanDataType::DATETIME:
        case HumanDataType::DATETIME_NULLABLE:
        case HumanDataType::URL:
        case HumanDataType::URL_NULLABLE:
        case HumanDataType::CURRENCY:
        case HumanDataType::CURRENCY_NULLABLE:
        case HumanDataType::TEXT:
          column_writers.emplace(
              col.first,
              new cstable::StringColumnWriter(0, 1));
          break;

        case HumanDataType::NULL_OR_EMPTY:
          column_writers.emplace(col.first, nullptr);
          break;

        case HumanDataType::UNKNOWN:
        case HumanDataType::BINARY:
          RAISEF(kTypeError, "invalid column type for column '$0'", col.first);

    }
  }

  csv->rewind();

  size_t num_records = 0;
  while (csv->readNextRow(&row)) {
    ++num_records;

    for (size_t i = 0; i < row.size() && i < columns.size(); ++i) {
      auto& col = column_writers[columns[i]];
      const auto& val = row[i];

      if (col.get() == nullptr) {
        continue;
      }

      if (Human::isNullOrEmpty(val)) {
        col->addNull(0, 0);
        continue;
      }

      switch (col->fieldType()) {

        case msg::FieldType::STRING: {
          col->addDatum(0, col->maxDefinitionLevel(), val.data(), val.size());
          break;
        }

        case msg::FieldType::UINT32: {
          uint32_t v = std::stoull(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::UINT64: {
          uint64_t v = std::stoull(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::DOUBLE: {
          double v = std::stod(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::BOOLEAN: {
          auto b = Human::parseBoolean(val);
          uint8_t v = !b.isEmpty() && b.get() ? 1 : 0;
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::OBJECT:
          RAISE(kIllegalStateError);

      }
    }
  }

  {
    cstable::CSTableWriter writer(flags.getString("output_file"), num_records);
    for (const auto& col : column_writers) {
      if (col.second.get()) {
        writer.addColumn(col.first, col.second.get());
      }
    }

    writer.commit();
  }
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  cli::CLI cli;

  /* command: from-csv */
  auto from_csv_cmd = cli.defineCommand("from-csv");
  from_csv_cmd->onCall(std::bind(&cmd_from_csv, std::placeholders::_1));

  from_csv_cmd->flags().defineFlag(
      "input_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "i",
      NULL,
      "input file",
      "<filename>");

  from_csv_cmd->flags().defineFlag(
      "output_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "o",
      NULL,
      "input file",
      "<filename>");

  cli.call(flags.getArgv());
  return 0;
}

<commit_msg>skip headers<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include "fnord/stdtypes.h"
#include "fnord/application.h"
#include "fnord/cli/flagparser.h"
#include "fnord/cli/CLI.h"
#include "fnord/csv/CSVInputStream.h"
#include "fnord/io/file.h"
#include "fnord/inspect.h"
#include "fnord/human.h"
#include "cstable/LEB128ColumnWriter.h"
#include "cstable/StringColumnWriter.h"
#include "cstable/DoubleColumnWriter.h"
#include "cstable/BooleanColumnWriter.h"
#include "cstable/CSTableWriter.h"

using namespace fnord;

void cmd_from_csv(const cli::FlagParser& flags) {
  auto csv = CSVInputStream::openFile(
      flags.getString("input_file"),
      '\t',
      '\n');

  Vector<String> columns;
  csv->readNextRow(&columns);

  HashMap<String, HumanDataType> column_types;
  for (const auto& hdr : columns) {
    column_types[hdr] = HumanDataType::UNKNOWN;
  }

  Vector<String> row;
  while (csv->readNextRow(&row)) {
    for (size_t i = 0; i < row.size() && i < columns.size(); ++i) {
      auto& ctype = column_types[columns[i]];
      ctype = Human::detectDataTypeSeries(row[i], ctype);
    }
  }

  HashMap<String, RefPtr<cstable::ColumnWriter>> column_writers;
  for (const auto& col : column_types) {
    switch (col.second) {

        case HumanDataType::UNSIGNED_INTEGER:
          column_writers.emplace(
              col.first,
              new cstable::LEB128ColumnWriter(0, 0));
          break;

        case HumanDataType::UNSIGNED_INTEGER_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::LEB128ColumnWriter(0, 1));
          break;

        case HumanDataType::SIGNED_INTEGER:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 0)); // FIXME
          break;

        case HumanDataType::SIGNED_INTEGER_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 1)); // FIXME
          break;

        case HumanDataType::FLOAT:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 0));
          break;

        case HumanDataType::FLOAT_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::DoubleColumnWriter(0, 1));
          break;

        case HumanDataType::BOOLEAN:
          column_writers.emplace(
              col.first,
              new cstable::BooleanColumnWriter(0, 0));
          break;

        case HumanDataType::BOOLEAN_NULLABLE:
          column_writers.emplace(
              col.first,
              new cstable::BooleanColumnWriter(0, 1));
          break;

        case HumanDataType::DATETIME:
        case HumanDataType::DATETIME_NULLABLE:
        case HumanDataType::URL:
        case HumanDataType::URL_NULLABLE:
        case HumanDataType::CURRENCY:
        case HumanDataType::CURRENCY_NULLABLE:
        case HumanDataType::TEXT:
          column_writers.emplace(
              col.first,
              new cstable::StringColumnWriter(0, 1));
          break;

        case HumanDataType::NULL_OR_EMPTY:
          column_writers.emplace(col.first, nullptr);
          break;

        case HumanDataType::UNKNOWN:
        case HumanDataType::BINARY:
          RAISEF(kTypeError, "invalid column type for column '$0'", col.first);

    }
  }

  csv->rewind();
  csv->skipNextRow();

  size_t num_records = 0;
  while (csv->readNextRow(&row)) {
    ++num_records;

    for (size_t i = 0; i < row.size() && i < columns.size(); ++i) {
      auto& col = column_writers[columns[i]];
      const auto& val = row[i];

      if (col.get() == nullptr) {
        continue;
      }

      if (Human::isNullOrEmpty(val)) {
        col->addNull(0, 0);
        continue;
      }

      switch (col->fieldType()) {

        case msg::FieldType::STRING: {
          col->addDatum(0, col->maxDefinitionLevel(), val.data(), val.size());
          break;
        }

        case msg::FieldType::UINT32: {
          uint32_t v = std::stoull(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::UINT64: {
          uint64_t v = std::stoull(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::DOUBLE: {
          double v = std::stod(val);
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::BOOLEAN: {
          auto b = Human::parseBoolean(val);
          uint8_t v = !b.isEmpty() && b.get() ? 1 : 0;
          col->addDatum(0, col->maxDefinitionLevel(), &v, sizeof(v));
          break;
        }

        case msg::FieldType::OBJECT:
          RAISE(kIllegalStateError);

      }
    }
  }

  {
    cstable::CSTableWriter writer(flags.getString("output_file"), num_records);
    for (const auto& col : column_writers) {
      if (col.second.get()) {
        writer.addColumn(col.first, col.second.get());
      }
    }

    writer.commit();
  }
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  cli::CLI cli;

  /* command: from-csv */
  auto from_csv_cmd = cli.defineCommand("from-csv");
  from_csv_cmd->onCall(std::bind(&cmd_from_csv, std::placeholders::_1));

  from_csv_cmd->flags().defineFlag(
      "input_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "i",
      NULL,
      "input file",
      "<filename>");

  from_csv_cmd->flags().defineFlag(
      "output_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "o",
      NULL,
      "input file",
      "<filename>");

  cli.call(flags.getArgv());
  return 0;
}

<|endoftext|>


<commit_before>#include <set>
#include <string>

#include <boost/asio.hpp>
#include <boost/bind.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/thread.hpp>

namespace asio = boost::asio;
namespace ip = asio::ip;
namespace placeholders = asio::placeholders;
namespace sys = boost::system;

class ConnectionManager;

class Connection
    : public boost::enable_shared_from_this< Connection >
{
public:

    Connection(
        asio::io_service & ioService,
        ConnectionManager & connectionManager
    )
        : m_socket( ioService )
        , m_strand( ioService )
        , m_connectionManager( connectionManager )
    {
    }

    ~Connection()
    {
    }

    ip::tcp::socket & socket()
    {
        return m_socket;
    }

    void start()
    {
        auto const onDataRead = boost::bind(
            & Connection::onDataRead,
            shared_from_this(),
            placeholders::error,
            placeholders::bytes_transferred()
        );

        m_socket.async_read_some(
            asio::buffer( m_request, m_maxLength ),
            m_strand.wrap( onDataRead )
        );
    }

    void onDataRead(
        sys::error_code const & errorCode,
        std::size_t const bytesTransferred
    )
    {
        if( errorCode )
        {
            std::cerr << "Response Read Error: " << errorCode.message() << std::endl;
        }
        else
        {
            std::cout.write( m_request, bytesTransferred );
            std::cout << std::endl;

            auto const onDataWritten = boost::bind(
                & Connection::onDataWritten,
                shared_from_this(),
                placeholders::error
            );

            asio::async_write(
                m_socket,
                asio::buffer( m_request, strlen( m_request ) ),
                onDataWritten
            );
        }
    }

    void onDataWritten(
        sys::error_code const & errorCode
    )
    {
        if( ! errorCode )
        {
            start();
        }
    }

    void onStop();

private:

    ip::tcp::socket m_socket;
    asio::io_service::strand m_strand;
    ConnectionManager & m_connectionManager;

    enum { m_maxLength = 1024 };
    char m_request[ m_maxLength ];
};

typedef boost::shared_ptr< Connection > ConnectionPtr;

class ConnectionManager
    : public boost::noncopyable
{
    typedef std::set< ConnectionPtr > ConnectionsPtr;

public:

    ConnectionsPtr::const_iterator begin() const
    {
        return m_connections.begin();
    }

    ConnectionsPtr::const_iterator cbegin() const
    {
        return m_connections.begin();
    }

    ConnectionsPtr::const_iterator end() const
    {
        return m_connections.end();
    }

    ConnectionsPtr::const_iterator cend() const
    {
        return m_connections.end();
    }

    void add(
        ConnectionPtr & connection
    )
    {
        m_connections.insert( connection );
    }

    void remove(
        ConnectionPtr const & connection
    )
    {
        auto const pos = m_connections.find( connection );

        if( pos != m_connections.end() ){
            m_connections.erase( pos );
        }
    }

private:

    ConnectionsPtr m_connections;
};

void Connection::onStop()
{
    char const * const message = "Goodbye.";

    auto const onDataWritten = boost::bind(
        & Connection::onDataWritten,
        shared_from_this(),
        placeholders::error
    );

    asio::async_write(
        m_socket,
        asio::buffer( message, strlen( message ) ),
        onDataWritten
    );

    m_connectionManager.remove( shared_from_this() );
}

class Server
{
public:

    Server(
        std::string const & port
    )
        : m_acceptor( m_ioService )
        , m_signals( m_ioService )
    {
        m_signals.add( SIGINT );
        m_signals.add( SIGTERM );
        m_signals.add( SIGQUIT );

        m_signals.async_wait(
            boost::bind( & Server::onStop, this )
        );

        ip::tcp::resolver resolver( m_ioService );
        ip::tcp::resolver::query query( "127.0.0.1", port );
        ip::tcp::endpoint endpoint = * resolver.resolve( query );

        m_acceptor.open( endpoint.protocol() );
        m_acceptor.set_option( ip::tcp::acceptor::reuse_address( true ) );
        m_acceptor.bind( endpoint );
        m_acceptor.listen();

        startAccept();
    }

    void run()
    {
        boost::thread_group threadGroup;

        auto const threadBody = boost::bind(
            & asio::io_service::run,
            & m_ioService
        );

        for( unsigned i = 0 ; i < 4 ; ++ i )
        {
            threadGroup.create_thread( threadBody );
        }

        threadGroup.join_all();
    }

private:

    void startAccept()
    {
        m_newConnection.reset( new Connection( m_ioService, m_connectionManager ) );

        m_connectionManager.add( m_newConnection );

        auto const onAccepted = boost::bind(
            & Server::onAccepted,
            this,
            placeholders::error
        );

        m_acceptor.async_accept(
            m_newConnection->socket(),
            onAccepted
        );
    }

    void onAccepted(
        sys::error_code const & errorCode
    )
    {
        if( ! errorCode )
        {
            m_newConnection->start();
        }

        startAccept();
    }

    void onStop()
    {
        for( auto const & connection : m_connectionManager )
        {
            connection->onStop();
        }

        m_ioService.stop();
    }

private:

    asio::io_service m_ioService;
    asio::ip::tcp::acceptor m_acceptor;
    asio::signal_set m_signals;

    ConnectionPtr m_newConnection;
    ConnectionManager m_connectionManager;
};

int main( int argc, char* argv[] )
{
    Server( argv[1] ).run();
}

<commit_msg>Close connections gracefully<commit_after>#include <set>
#include <string>

#include <boost/asio.hpp>
#include <boost/bind.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/shared_ptr.hpp>

#include <boost/thread.hpp>
#include <boost/thread/locks.hpp>

namespace asio = boost::asio;
namespace ip = asio::ip;
namespace placeholders = asio::placeholders;
namespace sys = boost::system;

class ConnectionManager;

class Connection
    : public boost::enable_shared_from_this< Connection >
{
public:

    Connection(
        asio::io_service & ioService,
        ConnectionManager & connectionManager
    )
        : m_socket( ioService )
        , m_strand( ioService )
        , m_connectionManager( connectionManager )
    {
    }

    ~Connection()
    {
    }

    ip::tcp::socket & socket()
    {
        return m_socket;
    }

    void start()
    {
        auto const onDataRead = boost::bind(
            & Connection::onDataRead,
            shared_from_this(),
            placeholders::error,
            placeholders::bytes_transferred()
        );

        m_socket.async_read_some(
            asio::buffer( m_request, m_maxLength ),
            m_strand.wrap( onDataRead )
        );
    }

    void onDataRead(
        sys::error_code const & errorCode,
        std::size_t const bytesTransferred
    )
    {
        if( errorCode )
        {
            std::cerr << "Response Read Error: " << errorCode.message() << std::endl;
        }
        else
        {
            std::cout.write( m_request, bytesTransferred );
            std::cout << std::endl;

            auto const onDataWritten = boost::bind(
                & Connection::onDataWritten,
                shared_from_this(),
                placeholders::error
            );

            asio::async_write(
                m_socket,
                asio::buffer( m_request, strlen( m_request ) ),
                onDataWritten
            );
        }
    }

    void onDataWritten(
        sys::error_code const & errorCode
    )
    {
        if( ! errorCode )
        {
            start();
        }
    }

    void onStop();

private:

    ip::tcp::socket m_socket;
    asio::io_service::strand m_strand;
    ConnectionManager & m_connectionManager;

    enum { m_maxLength = 1024 };
    char m_request[ m_maxLength ];
};

typedef boost::shared_ptr< Connection > ConnectionPtr;

class ConnectionManager
    : public boost::noncopyable
{
    typedef std::set< ConnectionPtr > ConnectionsPtr;

public:

    ConnectionsPtr::const_iterator begin() const
    {
        return m_connections.begin();
    }

    ConnectionsPtr::const_iterator cbegin() const
    {
        return m_connections.begin();
    }

    ConnectionsPtr::const_iterator end() const
    {
        return m_connections.end();
    }

    ConnectionsPtr::const_iterator cend() const
    {
        return m_connections.end();
    }

    template< typename Function, class... Args >
    void foreach( Function&& func, Args && ...args )
    {
        boost::lock_guard< boost::mutex > lock( m_mutex );

        for( auto & connection : m_connections )
        {
            func( connection, std::forward< Args >( args )... );
        }
    }

    void add(
        ConnectionPtr & connection
    )
    {
        boost::lock_guard< boost::mutex > lock( m_mutex );

        m_connections.insert( connection );
    }

    void remove(
        ConnectionPtr const & connection
    )
    {
        boost::lock_guard< boost::mutex > lock( m_mutex );

        auto const pos = m_connections.find( connection );

        if( pos != m_connections.end() ){
            m_connections.erase( pos );
        }
    }

private:

    boost::mutex m_mutex;
    ConnectionsPtr m_connections;
};

void Connection::onStop()
{
    char const * const message = "Goodbye.";

    auto const onDataWritten = boost::bind(
        & Connection::onDataWritten,
        shared_from_this(),
        placeholders::error
    );

    asio::async_write(
        m_socket,
        asio::buffer( message, strlen( message ) ),
        onDataWritten
    );

    m_connectionManager.remove( shared_from_this() );
}

class Server
{
public:

    Server(
        std::string const & port
    )
        : m_acceptor( m_ioService )
        , m_signals( m_ioService )
    {
        m_signals.add( SIGINT );
        m_signals.add( SIGTERM );
        m_signals.add( SIGQUIT );

        m_signals.async_wait(
            boost::bind( & Server::onStop, this )
        );

        ip::tcp::resolver resolver( m_ioService );
        ip::tcp::resolver::query query( "127.0.0.1", port );
        ip::tcp::endpoint endpoint = * resolver.resolve( query );

        m_acceptor.open( endpoint.protocol() );
        m_acceptor.set_option( ip::tcp::acceptor::reuse_address( true ) );
        m_acceptor.bind( endpoint );
        m_acceptor.listen();

        startAccept();
    }

    void run()
    {
        boost::thread_group threadGroup;

        auto const threadBody = boost::bind(
            & asio::io_service::run,
            & m_ioService
        );

        for( unsigned i = 0 ; i < 4 ; ++ i )
        {
            threadGroup.create_thread( threadBody );
        }

        threadGroup.join_all();
    }

private:

    void startAccept()
    {
        m_newConnection.reset( new Connection( m_ioService, m_connectionManager ) );

        m_connectionManager.add( m_newConnection );

        auto const onAccepted = boost::bind(
            & Server::onAccepted,
            this,
            placeholders::error
        );

        m_acceptor.async_accept(
            m_newConnection->socket(),
            onAccepted
        );
    }

    void onAccepted(
        sys::error_code const & errorCode
    )
    {
        if( ! errorCode )
        {
            m_newConnection->start();
        }

        startAccept();
    }

    void onStop()
    {
        m_connectionManager.foreach(
            boost::bind( & Connection::onStop, _1 )
        );

        m_ioService.stop();
    }

private:

    asio::io_service m_ioService;
    asio::ip::tcp::acceptor m_acceptor;
    asio::signal_set m_signals;

    ConnectionPtr m_newConnection;
    ConnectionManager m_connectionManager;
};

int main( int argc, char* argv[] )
{
    Server( argv[1] ).run();
}

<|endoftext|>


<commit_before>#include "cnn/nodes.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/gru.h"
#include "cnn/lstm.h"
#include "cnn/dict.h"
# include "cnn/expr.h"

#include <iostream>
#include <fstream>
#include <sstream>
#include <algorithm>

#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>

using namespace std;
using namespace cnn;

unsigned LAYERS = 2;
unsigned INPUT_DIM = 8;  //256
unsigned HIDDEN_DIM = 24;  // 1024
unsigned BATCH_SIZE = 4;
unsigned VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM});
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return Expression of total loss
  Expression BuildLMGraphs(const vector<vector<int> >& sents,
                           unsigned id,
                           unsigned bsize,
                           unsigned & chars,
                           ComputationGraph& cg) {
    const unsigned slen = sents[id].size();
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    Expression i_R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression i_bias = parameter(cg, p_bias);  // word bias
    vector<Expression> errs;
    vector<unsigned> last_arr(bsize, sents[0][0]), next_arr(bsize);
    for (unsigned t = 1; t < slen; ++t) {
      for (unsigned i = 0; i < bsize; ++i) {
        next_arr[i] = sents[id+i][t];
        if(next_arr[i] != *sents[id].rbegin()) chars++; // add non-EOS
      }
      // y_t = RNN(x_t)
      Expression i_x_t = lookup(cg, p_c, last_arr);
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;
      Expression i_err = pickneglogsoftmax(i_r_t, next_arr);
      errs.push_back(i_err);
      last_arr = next_arr;
    }
    Expression i_nerr = sum(errs);
    return i_nerr;
  }

  // return Expression for total loss
  void RandomSample(int max_len = 150) {
    cerr << endl;
    ComputationGraph cg;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();

    Expression i_R = parameter(cg, p_R);
    Expression i_bias = parameter(cg, p_bias);
    vector<Expression> errs;
    int len = 0;
    int cur = kSOS;
    while(len < max_len && cur != kEOS) {
      ++len;
      Expression i_x_t = lookup(cg, p_c, cur);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;

      Expression ydist = softmax(i_r_t);

      unsigned w = 0;
      while (w == 0 || (int)w == kSOS) {
        auto dist = as_vector(cg.incremental_forward());
        double p = rand01();
        for (; w < dist.size(); ++w) {
          p -= dist[w];
          if (p < 0.0) { break; }
        }
        if (w == dist.size()) w = kEOS;
      }
      cerr << (len == 1 ? "" : " ") << d.Convert(w);
      cur = w;
    }
    cerr << endl;
  }
};

// Sort in descending order of length
struct CompareLen {
  bool operator()(const std::vector<int>& first, const std::vector<int>& second) {
    return first.size() > second.size();
  }
};

int main(int argc, char** argv) {
  cnn::Initialize(argc, argv);
  if (argc != 3 && argc != 4) {
    cerr << "Usage: " << argv[0] << " corpus.txt dev.txt [model.params]\n";
    return 1;
  }
  kSOS = d.Convert("<s>");
  kEOS = d.Convert("</s>");
  vector<vector<int>> training, dev;
  string line;
  int tlc = 0;
  int ttoks = 0;
  cerr << "Reading training data from " << argv[1] << "...\n";
  {
    ifstream in(argv[1]);
    assert(in);
    while(getline(in, line)) {
      ++tlc;
      training.push_back(ReadSentence(line, &d));
      ttoks += training.back().size();
      if (training.back().front() != kSOS && training.back().back() != kEOS) {
        cerr << "Training sentence in " << argv[1] << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << tlc << " lines, " << ttoks << " tokens, " << d.size() << " types\n";
  }
  d.Freeze(); // no new word types allowed
  VOCAB_SIZE = d.size();

  // Sort the training sentences in descending order of length
  CompareLen comp;
  sort(training.begin(), training.end(), comp);
  // Pad the sentences in the same batch with EOS so they are the same length
  // This modifies the training objective a bit by making it necessary to
  // predict EOS multiple times, but it's easy and not so harmful
  for(size_t i = 0; i < training.size(); i += BATCH_SIZE)
      for(size_t j = 1; j < BATCH_SIZE; ++j)
        while(training[i+j].size() < training[i].size())
            training[i+j].push_back(kEOS);

  int dlc = 0;
  int dtoks = 0;
  cerr << "Reading dev data from " << argv[2] << "...\n";
  {
    ifstream in(argv[2]);
    assert(in);
    while(getline(in, line)) {
      ++dlc;
      dev.push_back(ReadSentence(line, &d));
      dtoks += dev.back().size();
      if (dev.back().front() != kSOS && dev.back().back() != kEOS) {
        cerr << "Dev sentence in " << argv[2] << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << dlc << " lines, " << dtoks << " tokens\n";
  }

  ostringstream os;
  os << "lm"
     << '_' << LAYERS
     << '_' << INPUT_DIM
     << '_' << HIDDEN_DIM
     << "-pid" << getpid() << ".params";
  const string fname = os.str();
  cerr << "Parameters will be written to: " << fname << endl;
  double best = 9e+99;

  Model model;
  Trainer* sgd = nullptr;
  sgd = new SimpleSGDTrainer(&model);
  sgd->clip_threshold *= BATCH_SIZE;

  RNNLanguageModel<LSTMBuilder> lm(model);
  if (argc == 4) {
    string fname = argv[3];
    ifstream in(fname);
    boost::archive::text_iarchive ia(in);
    ia >> model;
  }

  unsigned report_every_i = 50;
  unsigned dev_every_i_reports = 500;
  vector<unsigned> order((training.size()+BATCH_SIZE-1)/BATCH_SIZE);
  unsigned si = order.size();
  for (unsigned i = 0; i < order.size(); ++i) order[i] = i*BATCH_SIZE;
  bool first = true;
  int report = 0;
  unsigned lines = 0;
  while(1) {
    Timer iteration("completed in");
    double loss = 0;
    unsigned chars = 0;
    for (unsigned i = 0; i < report_every_i; ++i, ++si) {
      if (si == order.size()) {
        si = 0;
        if (first) { first = false; } else { sgd->update_epoch(); }
        cerr << "**SHUFFLE\n";
        shuffle(order.begin(), order.end(), *rndeng);
      }
      // build graph for this instance
      ComputationGraph cg;
      unsigned bsize = std::min((unsigned)training.size()-order[si], BATCH_SIZE); // Batch size
      lm.BuildLMGraphs(training, order[si], bsize, chars, cg);
      vector<float> losses = as_vector(cg.forward());
      for(float l : losses)
        loss += l;
      cg.backward();
      sgd->update();
      lines += bsize;
    }
    sgd->status();
    cerr << " E = " << (loss / chars) << " ppl=" << exp(loss / chars) << ' ';
    lm.RandomSample();

    // show score on dev data?
    report++;
    if (report % dev_every_i_reports == 0) {
      double dloss = 0;
      unsigned dchars = 0;
      for (unsigned i = 0; i < dev.size(); ++i) {
        ComputationGraph cg;
        lm.BuildLMGraphs(dev, i, 1, dchars, cg);
        dloss += as_scalar(cg.forward());
      }
      if (dloss < best) {
        best = dloss;
        ofstream out(fname);
        boost::archive::text_oarchive oa(out);
        oa << model;
      }
      cerr << "\n***DEV [epoch=" << (lines / (double)training.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
    }
  }
  delete sgd;
}

<commit_msg>Added sum_batches to rnnlm-batch<commit_after>#include "cnn/nodes.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/gru.h"
#include "cnn/lstm.h"
#include "cnn/dict.h"
# include "cnn/expr.h"

#include <iostream>
#include <fstream>
#include <sstream>
#include <algorithm>

#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>

using namespace std;
using namespace cnn;

unsigned LAYERS = 2;
unsigned INPUT_DIM = 8;  //256
unsigned HIDDEN_DIM = 24;  // 1024
unsigned BATCH_SIZE = 4;
unsigned VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM});
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return Expression of total loss
  Expression BuildLMGraphs(const vector<vector<int> >& sents,
                           unsigned id,
                           unsigned bsize,
                           unsigned & chars,
                           ComputationGraph& cg) {
    const unsigned slen = sents[id].size();
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    Expression i_R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression i_bias = parameter(cg, p_bias);  // word bias
    vector<Expression> errs;
    vector<unsigned> last_arr(bsize, sents[0][0]), next_arr(bsize);
    for (unsigned t = 1; t < slen; ++t) {
      for (unsigned i = 0; i < bsize; ++i) {
        next_arr[i] = sents[id+i][t];
        if(next_arr[i] != *sents[id].rbegin()) chars++; // add non-EOS
      }
      // y_t = RNN(x_t)
      Expression i_x_t = lookup(cg, p_c, last_arr);
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;
      Expression i_err = pickneglogsoftmax(i_r_t, next_arr);
      errs.push_back(i_err);
      last_arr = next_arr;
    }
    Expression i_nerr = sum_batches(sum(errs));
    return i_nerr;
  }

  // return Expression for total loss
  void RandomSample(int max_len = 150) {
    cerr << endl;
    ComputationGraph cg;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();

    Expression i_R = parameter(cg, p_R);
    Expression i_bias = parameter(cg, p_bias);
    vector<Expression> errs;
    int len = 0;
    int cur = kSOS;
    while(len < max_len && cur != kEOS) {
      ++len;
      Expression i_x_t = lookup(cg, p_c, cur);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;

      Expression ydist = softmax(i_r_t);

      unsigned w = 0;
      while (w == 0 || (int)w == kSOS) {
        auto dist = as_vector(cg.incremental_forward());
        double p = rand01();
        for (; w < dist.size(); ++w) {
          p -= dist[w];
          if (p < 0.0) { break; }
        }
        if (w == dist.size()) w = kEOS;
      }
      cerr << (len == 1 ? "" : " ") << d.Convert(w);
      cur = w;
    }
    cerr << endl;
  }
};

// Sort in descending order of length
struct CompareLen {
  bool operator()(const std::vector<int>& first, const std::vector<int>& second) {
    return first.size() > second.size();
  }
};

int main(int argc, char** argv) {
  cnn::Initialize(argc, argv);
  if (argc != 3 && argc != 4) {
    cerr << "Usage: " << argv[0] << " corpus.txt dev.txt [model.params]\n";
    return 1;
  }
  kSOS = d.Convert("<s>");
  kEOS = d.Convert("</s>");
  vector<vector<int>> training, dev;
  string line;
  int tlc = 0;
  int ttoks = 0;
  cerr << "Reading training data from " << argv[1] << "...\n";
  {
    ifstream in(argv[1]);
    assert(in);
    while(getline(in, line)) {
      ++tlc;
      training.push_back(ReadSentence(line, &d));
      ttoks += training.back().size();
      if (training.back().front() != kSOS && training.back().back() != kEOS) {
        cerr << "Training sentence in " << argv[1] << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << tlc << " lines, " << ttoks << " tokens, " << d.size() << " types\n";
  }
  d.Freeze(); // no new word types allowed
  VOCAB_SIZE = d.size();

  // Sort the training sentences in descending order of length
  CompareLen comp;
  sort(training.begin(), training.end(), comp);
  // Pad the sentences in the same batch with EOS so they are the same length
  // This modifies the training objective a bit by making it necessary to
  // predict EOS multiple times, but it's easy and not so harmful
  for(size_t i = 0; i < training.size(); i += BATCH_SIZE)
      for(size_t j = 1; j < BATCH_SIZE; ++j)
        while(training[i+j].size() < training[i].size())
            training[i+j].push_back(kEOS);

  int dlc = 0;
  int dtoks = 0;
  cerr << "Reading dev data from " << argv[2] << "...\n";
  {
    ifstream in(argv[2]);
    assert(in);
    while(getline(in, line)) {
      ++dlc;
      dev.push_back(ReadSentence(line, &d));
      dtoks += dev.back().size();
      if (dev.back().front() != kSOS && dev.back().back() != kEOS) {
        cerr << "Dev sentence in " << argv[2] << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << dlc << " lines, " << dtoks << " tokens\n";
  }

  ostringstream os;
  os << "lm"
     << '_' << LAYERS
     << '_' << INPUT_DIM
     << '_' << HIDDEN_DIM
     << "-pid" << getpid() << ".params";
  const string fname = os.str();
  cerr << "Parameters will be written to: " << fname << endl;
  double best = 9e+99;

  Model model;
  Trainer* sgd = nullptr;
  sgd = new SimpleSGDTrainer(&model);
  sgd->clip_threshold *= BATCH_SIZE;

  RNNLanguageModel<LSTMBuilder> lm(model);
  if (argc == 4) {
    string fname = argv[3];
    ifstream in(fname);
    boost::archive::text_iarchive ia(in);
    ia >> model;
  }

  unsigned report_every_i = 50;
  unsigned dev_every_i_reports = 500;
  vector<unsigned> order((training.size()+BATCH_SIZE-1)/BATCH_SIZE);
  unsigned si = order.size();
  for (unsigned i = 0; i < order.size(); ++i) order[i] = i*BATCH_SIZE;
  bool first = true;
  int report = 0;
  unsigned lines = 0;
  while(1) {
    Timer iteration("completed in");
    double loss = 0;
    unsigned chars = 0;
    for (unsigned i = 0; i < report_every_i; ++i, ++si) {
      if (si == order.size()) {
        si = 0;
        if (first) { first = false; } else { sgd->update_epoch(); }
        cerr << "**SHUFFLE\n";
        shuffle(order.begin(), order.end(), *rndeng);
      }
      // build graph for this instance
      ComputationGraph cg;
      unsigned bsize = std::min((unsigned)training.size()-order[si], BATCH_SIZE); // Batch size
      lm.BuildLMGraphs(training, order[si], bsize, chars, cg);
      loss += as_scalar(cg.forward());
      cg.backward();
      sgd->update();
      lines += bsize;
    }
    sgd->status();
    cerr << " E = " << (loss / chars) << " ppl=" << exp(loss / chars) << ' ';
    lm.RandomSample();

    // show score on dev data?
    report++;
    if (report % dev_every_i_reports == 0) {
      double dloss = 0;
      unsigned dchars = 0;
      for (unsigned i = 0; i < dev.size(); ++i) {
        ComputationGraph cg;
        lm.BuildLMGraphs(dev, i, 1, dchars, cg);
        dloss += as_scalar(cg.forward());
      }
      if (dloss < best) {
        best = dloss;
        ofstream out(fname);
        boost::archive::text_oarchive oa(out);
        oa << model;
      }
      cerr << "\n***DEV [epoch=" << (lines / (double)training.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
    }
  }
  delete sgd;
}

<|endoftext|>


<commit_before>#pragma once

#include <memory>

#include "Runtime/CRandom16.hpp"
#include "Runtime/CToken.hpp"
#include "Runtime/GCNTypes.hpp"

namespace urde {
class CSimplePool;
class CTransitionDatabaseGame;

struct CAnimSysContext {
  TToken<CTransitionDatabaseGame> x0_transDB;
  std::shared_ptr<CRandom16> x8_random;
  CSimplePool& xc_store;

  CAnimSysContext(const TToken<CTransitionDatabaseGame>& transDB, u32 randomSeed, CSimplePool& store)
  : x0_transDB(transDB), x8_random(std::make_shared<CRandom16>(randomSeed)), xc_store(store) {}
};

} // namespace urde
<commit_msg>CAnimSysContext: std::move TToken instance in constructor<commit_after>#pragma once

#include <memory>

#include "Runtime/CRandom16.hpp"
#include "Runtime/CToken.hpp"
#include "Runtime/GCNTypes.hpp"

namespace urde {
class CSimplePool;
class CTransitionDatabaseGame;

struct CAnimSysContext {
  TToken<CTransitionDatabaseGame> x0_transDB;
  std::shared_ptr<CRandom16> x8_random;
  CSimplePool& xc_store;

  CAnimSysContext(TToken<CTransitionDatabaseGame> transDB, u32 randomSeed, CSimplePool& store)
  : x0_transDB(std::move(transDB)), x8_random(std::make_shared<CRandom16>(randomSeed)), xc_store(store) {}
};

} // namespace urde
<|endoftext|>


<commit_before>/*
 *  DataFileIndexer.cp
 *  DataFileIndexer
 *
 *  Created by bkennedy on 2/19/08.
 *  Copyright 2008 MIT. All rights reserved.
 *
 */

#include "DataFileIndexer.h"
#include "DataFileUtilities.h"
#include <boost/shared_ptr.hpp>
#include <algorithm>
#include <iostream>

#include "ScarabServices.h"

BEGIN_NAMESPACE_MW
BEGIN_NAMESPACE(scarab)

DataFileIndexer::~DataFileIndexer() {
    scarab_session_close(session);
}

DataFileIndexer::DataFileIndexer(const boost::filesystem::path &data_file) :
    number_of_events(0),
    events_per_block(0)
{
    const std::string uri("ldobinary:file_readonly://" + data_file.string());
    session = scarab_session_connect(uri.c_str());
    if ((session == NULL) || (scarab_session_geterr(session) != 0)) {
        scarab_mem_free(session);
		throw DataFileIndexerError(boost::format("Cannot open file \"%s\"") % data_file.string());
    }
}

void DataFileIndexer::buildIndex(unsigned int _events_per_block, unsigned int multiplication_factor_per_level) {
    number_of_events = 0;
    events_per_block = _events_per_block;
    root.reset();

	{
		std::vector<boost::shared_ptr<EventBlock> > event_blocks;
		{
			std::set<unsigned int> event_codes_in_block;
			MWTime max_time = MIN_MWORKS_TIME();
			MWTime min_time = MAX_MWORKS_TIME();
			long int previous_datum_location = scarab_tell(session);

			ScarabDatum *datum = NULL;
			while(datum = scarab_read(session)) {
                if (!isScarabEvent(datum)) {
                    // Ignore invalid events
                    scarab_free_datum(datum);
                    continue;
                }

				event_codes_in_block.insert(getScarabEventCode(datum));

				MWTime event_time = getScarabEventTime(datum);
				max_time = std::max(max_time, event_time);
				min_time = std::min(min_time, event_time);

				number_of_events++;
				if(number_of_events % events_per_block == 0) {
					boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
					event_blocks.push_back(new_event_block);

					event_codes_in_block.clear();
					max_time = MIN_MWORKS_TIME();
					min_time = MAX_MWORKS_TIME();
					previous_datum_location = scarab_tell(session);
				}
				scarab_free_datum(datum);
			}

			// add in the remainder blocks
			boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
			event_blocks.push_back(new_event_block);
		}

		{
			// build the tree
			int events_per_node = events_per_block;
			int number_of_levels = 1;
			while(events_per_node < number_of_events) {
				number_of_levels++;
				events_per_node *= multiplication_factor_per_level;
			}

			std::vector <boost::shared_ptr<EventBlock> > blocks_at_next_level = event_blocks;
			for(int i = 1; i < number_of_levels; ++i) {
				std::vector <boost::shared_ptr<EventBlock> > blocks_at_current_level;

				std::vector<boost::shared_ptr<EventBlock> >::const_iterator j = blocks_at_next_level.begin();
				while(j != blocks_at_next_level.end()) {
					std::vector <boost::shared_ptr<EventBlock> > children;
					for(int k = 0; k < multiplication_factor_per_level && j != blocks_at_next_level.end(); ++k) {
						children.push_back(*j);
						++j;
					}
					boost::shared_ptr<EventBlock> new_block = boost::shared_ptr<EventBlock>(new EventBlock(children));
					blocks_at_current_level.push_back(new_block);
				}

				blocks_at_next_level = blocks_at_current_level;
			}

			if(blocks_at_next_level.size() != 1) {
				//badness
				throw DataFileIndexerError("Something went wrong ... please abort and try again");
			}

            // DDC added a patch to fix failure to index small numbers of events
            // force a mandatory level below root
            boost::shared_ptr<EventBlock> subroot = blocks_at_next_level.at(0);
            std::vector< boost::shared_ptr<EventBlock> > root_children;
            root_children.push_back(subroot);
			root = boost::shared_ptr<EventBlock>(new EventBlock(root_children));
        }
	}
}

DataFileIndexer::EventsIterator::EventsIterator(const DataFileIndexer &_dfi,
                                                const std::set<unsigned int> &_event_codes_to_match,
                                                MWTime _lower_bound,
                                                MWTime _upper_bound) :
    dfi(_dfi),
    event_codes_to_match(_event_codes_to_match),
    lower_bound(_lower_bound),
    upper_bound(_upper_bound)
{
    if (lower_bound > upper_bound) {
        throw DataFileIndexerError("Minimum event time must be less than or equal to maximum event time");
    }

    // Recursively find event blocks that meet our search criteria
	dfi.root->children(matching_event_blocks, event_codes_to_match, lower_bound, upper_bound);

    // Prepare for iteration
    current_event_block = 0;
    current_relative_event = dfi.events_per_block;
    current_datum = NULL;
}

bool DataFileIndexer::EventsIterator::getNextEvent(int &code, MWTime &time, Datum &data) {
    bool foundNextEvent = false;

    while (!foundNextEvent && (current_event_block < matching_event_blocks.size())) {
        if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL)) {
            // Advance to the next block
            scarab_seek(dfi.session, matching_event_blocks.at(current_event_block)->blockOffset(), SEEK_SET);
            current_relative_event = 0;
        }

        // Read through the event block
		while (!foundNextEvent && (current_relative_event < dfi.events_per_block) && (current_datum = scarab_read(dfi.session)))
        {
            if (!isScarabEvent(current_datum)) {
                // Skip invalid events
                scarab_free_datum(current_datum);
                continue;
            }

			MWTime event_time = getScarabEventTime(current_datum);

            // Check the time criterion
            if (event_time >= lower_bound && event_time <= upper_bound) {
				unsigned int event_code = getScarabEventCode(current_datum);

                // Check if the event code matches
                if (event_codes_to_match.empty() ||
                    (event_codes_to_match.find(event_code) != event_codes_to_match.end()))
                {
                    code = event_code;
                    time = event_time;
                    data = scarabDatumToDatum(getScarabEventPayload(current_datum));
                    foundNextEvent = true;
                }
			}

			scarab_free_datum(current_datum);
			current_relative_event++;
		}

        if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL))
            current_event_block++;
    }

    return foundNextEvent;
}

END_NAMESPACE(scarab)
END_NAMESPACE_MW
<commit_msg>DataFileIndexer constructor now ensures that scarab_init is called<commit_after>/*
 *  DataFileIndexer.cp
 *  DataFileIndexer
 *
 *  Created by bkennedy on 2/19/08.
 *  Copyright 2008 MIT. All rights reserved.
 *
 */

#include "DataFileIndexer.h"
#include "DataFileUtilities.h"
#include <boost/shared_ptr.hpp>
#include <algorithm>
#include <iostream>

#include "ScarabServices.h"

BEGIN_NAMESPACE_MW
BEGIN_NAMESPACE(scarab)

DataFileIndexer::~DataFileIndexer() {
    scarab_session_close(session);
}

DataFileIndexer::DataFileIndexer(const boost::filesystem::path &data_file) :
    number_of_events(0),
    events_per_block(0)
{
    // Perform one-time initialization of Scarab library
    static std::once_flag scarabInitFlag;
    std::call_once(scarabInitFlag, []() {
        if (scarab_init(0) != 0) {
            throw DataFileIndexerError("Scarab initialization failed");
        }
    });

    const std::string uri("ldobinary:file_readonly://" + data_file.string());
    session = scarab_session_connect(uri.c_str());
    if ((session == NULL) || (scarab_session_geterr(session) != 0)) {
        scarab_mem_free(session);
		throw DataFileIndexerError(boost::format("Cannot open file \"%s\"") % data_file.string());
    }
}

void DataFileIndexer::buildIndex(unsigned int _events_per_block, unsigned int multiplication_factor_per_level) {
    number_of_events = 0;
    events_per_block = _events_per_block;
    root.reset();

	{
		std::vector<boost::shared_ptr<EventBlock> > event_blocks;
		{
			std::set<unsigned int> event_codes_in_block;
			MWTime max_time = MIN_MWORKS_TIME();
			MWTime min_time = MAX_MWORKS_TIME();
			long int previous_datum_location = scarab_tell(session);

			ScarabDatum *datum = NULL;
			while(datum = scarab_read(session)) {
                if (!isScarabEvent(datum)) {
                    // Ignore invalid events
                    scarab_free_datum(datum);
                    continue;
                }

				event_codes_in_block.insert(getScarabEventCode(datum));

				MWTime event_time = getScarabEventTime(datum);
				max_time = std::max(max_time, event_time);
				min_time = std::min(min_time, event_time);

				number_of_events++;
				if(number_of_events % events_per_block == 0) {
					boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
					event_blocks.push_back(new_event_block);

					event_codes_in_block.clear();
					max_time = MIN_MWORKS_TIME();
					min_time = MAX_MWORKS_TIME();
					previous_datum_location = scarab_tell(session);
				}
				scarab_free_datum(datum);
			}

			// add in the remainder blocks
			boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
			event_blocks.push_back(new_event_block);
		}

		{
			// build the tree
			int events_per_node = events_per_block;
			int number_of_levels = 1;
			while(events_per_node < number_of_events) {
				number_of_levels++;
				events_per_node *= multiplication_factor_per_level;
			}

			std::vector <boost::shared_ptr<EventBlock> > blocks_at_next_level = event_blocks;
			for(int i = 1; i < number_of_levels; ++i) {
				std::vector <boost::shared_ptr<EventBlock> > blocks_at_current_level;

				std::vector<boost::shared_ptr<EventBlock> >::const_iterator j = blocks_at_next_level.begin();
				while(j != blocks_at_next_level.end()) {
					std::vector <boost::shared_ptr<EventBlock> > children;
					for(int k = 0; k < multiplication_factor_per_level && j != blocks_at_next_level.end(); ++k) {
						children.push_back(*j);
						++j;
					}
					boost::shared_ptr<EventBlock> new_block = boost::shared_ptr<EventBlock>(new EventBlock(children));
					blocks_at_current_level.push_back(new_block);
				}

				blocks_at_next_level = blocks_at_current_level;
			}

			if(blocks_at_next_level.size() != 1) {
				//badness
				throw DataFileIndexerError("Something went wrong ... please abort and try again");
			}

            // DDC added a patch to fix failure to index small numbers of events
            // force a mandatory level below root
            boost::shared_ptr<EventBlock> subroot = blocks_at_next_level.at(0);
            std::vector< boost::shared_ptr<EventBlock> > root_children;
            root_children.push_back(subroot);
			root = boost::shared_ptr<EventBlock>(new EventBlock(root_children));
        }
	}
}

DataFileIndexer::EventsIterator::EventsIterator(const DataFileIndexer &_dfi,
                                                const std::set<unsigned int> &_event_codes_to_match,
                                                MWTime _lower_bound,
                                                MWTime _upper_bound) :
    dfi(_dfi),
    event_codes_to_match(_event_codes_to_match),
    lower_bound(_lower_bound),
    upper_bound(_upper_bound)
{
    if (lower_bound > upper_bound) {
        throw DataFileIndexerError("Minimum event time must be less than or equal to maximum event time");
    }

    // Recursively find event blocks that meet our search criteria
	dfi.root->children(matching_event_blocks, event_codes_to_match, lower_bound, upper_bound);

    // Prepare for iteration
    current_event_block = 0;
    current_relative_event = dfi.events_per_block;
    current_datum = NULL;
}

bool DataFileIndexer::EventsIterator::getNextEvent(int &code, MWTime &time, Datum &data) {
    bool foundNextEvent = false;

    while (!foundNextEvent && (current_event_block < matching_event_blocks.size())) {
        if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL)) {
            // Advance to the next block
            scarab_seek(dfi.session, matching_event_blocks.at(current_event_block)->blockOffset(), SEEK_SET);
            current_relative_event = 0;
        }

        // Read through the event block
		while (!foundNextEvent && (current_relative_event < dfi.events_per_block) && (current_datum = scarab_read(dfi.session)))
        {
            if (!isScarabEvent(current_datum)) {
                // Skip invalid events
                scarab_free_datum(current_datum);
                continue;
            }

			MWTime event_time = getScarabEventTime(current_datum);

            // Check the time criterion
            if (event_time >= lower_bound && event_time <= upper_bound) {
				unsigned int event_code = getScarabEventCode(current_datum);

                // Check if the event code matches
                if (event_codes_to_match.empty() ||
                    (event_codes_to_match.find(event_code) != event_codes_to_match.end()))
                {
                    code = event_code;
                    time = event_time;
                    data = scarabDatumToDatum(getScarabEventPayload(current_datum));
                    foundNextEvent = true;
                }
			}

			scarab_free_datum(current_datum);
			current_relative_event++;
		}

        if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL))
            current_event_block++;
    }

    return foundNextEvent;
}

END_NAMESPACE(scarab)
END_NAMESPACE_MW
<|endoftext|>


<commit_before>#include "musicidentifysongsthread.h"
#include "musicsourcedownloadthread.h"
#///QJson import
#include "qjson/parser.h"

#include <QEventLoop>
#include <QCryptographicHash>

#define QUERY_URL     "http://ap-southeast-1.api.acrcloud.com/v1/identify"
#define ACRUA_URL     "http://7xpa0g.com1.z0.glb.clouddn.com/acrcloud"

MusicIdentifySongsThread::MusicIdentifySongsThread(QObject *parent)
    : MusicNetworkAbstract(parent)
{
    m_manager = new QNetworkAccessManager(this);
#ifndef QT_NO_SSL
    connect(m_manager, SIGNAL(sslErrors(QNetworkReply*,QList<QSslError>)),
                       SLOT(sslErrors(QNetworkReply*,QList<QSslError>)));
    M_LOGGER_INFO(QString("%1 Support ssl: %2").arg(getClassName()).arg(QSslSocket::supportsSsl()));
#endif
}

QString MusicIdentifySongsThread::getClassName()
{
    return staticMetaObject.className();
}

bool MusicIdentifySongsThread::getKey()
{
    QEventLoop loop;
    connect(this, SIGNAL(getKeyFinished()), &loop, SLOT(quit()));

    MusicSourceDownloadThread *download = new MusicSourceDownloadThread(this);
    connect(download, SIGNAL(downLoadByteDataChanged(QByteArray)), SLOT(keyDownLoadFinished(QByteArray)));
    download->startToDownload(ACRUA_URL);

    loop.exec();

    return !m_accessKey.isEmpty() && !m_accessSecret.isEmpty();
}

void MusicIdentifySongsThread::query(const QString &path)
{
    QString boundary = "----";
    QString startBoundary = "--" + boundary;
    QString endBoundary = "\r\n--" + boundary + "--\r\n";
    QString contentType = "multipart/form-data; boundary=" + boundary;

    QString httpMethod = "POST";
    QString httpUri = "/v1/identify";
    QString dataType = "audio";
    QString signatureVersion = "1";
    QString timeStamp = QString::number(QDateTime::currentDateTime().toMSecsSinceEpoch());

    httpMethod = httpMethod + "\n" + httpUri + "\n" + m_accessKey + "\n" + dataType + "\n" + signatureVersion + "\n" + timeStamp;
    QByteArray content = hmacSha1(httpMethod.toUtf8(), m_accessSecret.toUtf8()).toBase64();

    QString value;
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"access_key\"\r\n\r\n" + m_accessKey + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"data_type\"\r\n\r\n" + dataType + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"timestamp\"\r\n\r\n" + timeStamp + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"signature_version\"\r\n\r\n" + signatureVersion + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"signature\"\r\n\r\n" + content + "\r\n";

    QFile file(path);
    if(!file.open(QFile::ReadOnly))
    {
        return;
    }
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"sample_bytes\"\r\n\r\n" + QString::number(file.size()) + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"sample\"; filename=\"" + file.fileName() + "\"\r\n";
    value += "Content-Type: application/octet-stream\r\n\r\n";

    content.clear();
    content.append(value.toUtf8());
    content.append(file.readAll());
    content.append(endBoundary);

    QNetworkRequest request;
    request.setUrl(QUrl(QUERY_URL));
    request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
#ifndef QT_NO_SSL
    QSslConfiguration sslConfig = request.sslConfiguration();
    sslConfig.setPeerVerifyMode(QSslSocket::VerifyNone);
    request.setSslConfiguration(sslConfig);
#endif
    m_reply = m_manager->post(request, content);
    connect(m_reply, SIGNAL(finished()), SLOT(downLoadFinished()));
}

const MusicSongIdentifys& MusicIdentifySongsThread::getIdentifySongs() const
{
    return m_songIdentifys;
}

void MusicIdentifySongsThread::downLoadFinished()
{
    if(m_reply && m_reply->error() == QNetworkReply::NoError)
    {
        QJson::Parser parser;
        bool ok;
        QVariant data = parser.parse(m_reply->readAll(), &ok);
        if(ok)
        {
            QVariantMap value = data.toMap();
            if(value.contains("metadata"))
            {
                value = value["metadata"].toMap();
                QVariantList list = value["music"].toList();
                foreach(const QVariant &var, list)
                {
                    value = var.toMap();

                    MusicSongIdentify song;
                    song.m_songName = value["title"].toString();
                    foreach(const QVariant &artists, value["artists"].toList())
                    {
                        value = artists.toMap();
                        song.m_singerName = value["name"].toString();
                        break;
                    }
                    m_songIdentifys << song;
                    break;
                }
            }
        }
    }

    emit downLoadDataChanged( QString() );
    if(m_reply)
    {
        m_reply->deleteLater();
        m_reply = nullptr;
    }
}

void MusicIdentifySongsThread::keyDownLoadFinished(const QByteArray &data)
{
    QJson::Parser parser;
    bool ok;
    QVariant dt = parser.parse(data, &ok);
    if(ok)
    {
        QVariantMap value = dt.toMap();
        if( QDateTime::fromString( value["time"].toString(), "yyyy-MM-dd HH:mm:ss") >
            QDateTime::currentDateTime())
        {
            m_accessKey = value["key"].toString();
            m_accessSecret = value["secret"].toString();
        }
    }
    emit getKeyFinished();
}

QByteArray MusicIdentifySongsThread::hmacSha1(const QByteArray &data, const QByteArray &secretKey)
{
    int blockSize = 64;
    QByteArray newSecretKey = secretKey;
    if(newSecretKey.length() > blockSize)
    {
        newSecretKey = QCryptographicHash::hash(newSecretKey, QCryptographicHash::Sha1);
    }

    QByteArray innerPadding(blockSize, char(0x36));
    QByteArray outerPadding(blockSize, char(0x5c));

    for(int i = 0; i < secretKey.length(); i++)
    {
        innerPadding[i] = innerPadding[i] ^ secretKey.at(i);
        outerPadding[i] = outerPadding[i] ^ secretKey.at(i);
    }

    QByteArray total = outerPadding;
    QByteArray part = innerPadding;
    part.append(data);
    total.append(QCryptographicHash::hash(part, QCryptographicHash::Sha1));
    QByteArray hashed = QCryptographicHash::hash(total, QCryptographicHash::Sha1);
    return hashed;
}
<commit_msg>fix song identify error in undefined songs[963201]<commit_after>#include "musicidentifysongsthread.h"
#include "musicsourcedownloadthread.h"
#///QJson import
#include "qjson/parser.h"

#include <QEventLoop>
#include <QCryptographicHash>

#define QUERY_URL     "http://ap-southeast-1.api.acrcloud.com/v1/identify"
#define ACRUA_URL     "http://7xpa0g.com1.z0.glb.clouddn.com/acrcloud"

MusicIdentifySongsThread::MusicIdentifySongsThread(QObject *parent)
    : MusicNetworkAbstract(parent)
{
    m_manager = new QNetworkAccessManager(this);
#ifndef QT_NO_SSL
    connect(m_manager, SIGNAL(sslErrors(QNetworkReply*,QList<QSslError>)),
                       SLOT(sslErrors(QNetworkReply*,QList<QSslError>)));
    M_LOGGER_INFO(QString("%1 Support ssl: %2").arg(getClassName()).arg(QSslSocket::supportsSsl()));
#endif
}

QString MusicIdentifySongsThread::getClassName()
{
    return staticMetaObject.className();
}

bool MusicIdentifySongsThread::getKey()
{
    QEventLoop loop;
    connect(this, SIGNAL(getKeyFinished()), &loop, SLOT(quit()));

    MusicSourceDownloadThread *download = new MusicSourceDownloadThread(this);
    connect(download, SIGNAL(downLoadByteDataChanged(QByteArray)), SLOT(keyDownLoadFinished(QByteArray)));
    download->startToDownload(ACRUA_URL);

    loop.exec();

    return !m_accessKey.isEmpty() && !m_accessSecret.isEmpty();
}

void MusicIdentifySongsThread::query(const QString &path)
{
    QString boundary = "----";
    QString startBoundary = "--" + boundary;
    QString endBoundary = "\r\n--" + boundary + "--\r\n";
    QString contentType = "multipart/form-data; boundary=" + boundary;

    QString httpMethod = "POST";
    QString httpUri = "/v1/identify";
    QString dataType = "audio";
    QString signatureVersion = "1";
    QString timeStamp = QString::number(QDateTime::currentDateTime().toMSecsSinceEpoch());

    httpMethod = httpMethod + "\n" + httpUri + "\n" + m_accessKey + "\n" + dataType + "\n" + signatureVersion + "\n" + timeStamp;
    QByteArray content = hmacSha1(httpMethod.toUtf8(), m_accessSecret.toUtf8()).toBase64();

    QString value;
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"access_key\"\r\n\r\n" + m_accessKey + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"data_type\"\r\n\r\n" + dataType + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"timestamp\"\r\n\r\n" + timeStamp + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"signature_version\"\r\n\r\n" + signatureVersion + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"signature\"\r\n\r\n" + content + "\r\n";

    QFile file(path);
    if(!file.open(QFile::ReadOnly))
    {
        return;
    }
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"sample_bytes\"\r\n\r\n" + QString::number(file.size()) + "\r\n";
    value += startBoundary + "\r\nContent-Disposition: form-data; name=\"sample\"; filename=\"" + file.fileName() + "\"\r\n";
    value += "Content-Type: application/octet-stream\r\n\r\n";

    content.clear();
    content.append(value.toUtf8());
    content.append(file.readAll());
    content.append(endBoundary);

    QNetworkRequest request;
    request.setUrl(QUrl(QUERY_URL));
    request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
#ifndef QT_NO_SSL
    QSslConfiguration sslConfig = request.sslConfiguration();
    sslConfig.setPeerVerifyMode(QSslSocket::VerifyNone);
    request.setSslConfiguration(sslConfig);
#endif
    m_reply = m_manager->post(request, content);
    connect(m_reply, SIGNAL(finished()), SLOT(downLoadFinished()));
}

const MusicSongIdentifys& MusicIdentifySongsThread::getIdentifySongs() const
{
    return m_songIdentifys;
}

void MusicIdentifySongsThread::downLoadFinished()
{
    m_songIdentifys.clear();
    if(m_reply && m_reply->error() == QNetworkReply::NoError)
    {
        QJson::Parser parser;
        bool ok;
        QVariant data = parser.parse(m_reply->readAll(), &ok);
        if(ok)
        {
            QVariantMap value = data.toMap();
            if(value.contains("metadata"))
            {
                value = value["metadata"].toMap();
                QVariantList list = value["music"].toList();
                foreach(const QVariant &var, list)
                {
                    value = var.toMap();

                    MusicSongIdentify song;
                    song.m_songName = value["title"].toString();
                    foreach(const QVariant &artists, value["artists"].toList())
                    {
                        value = artists.toMap();
                        song.m_singerName = value["name"].toString();
                        break;
                    }
                    m_songIdentifys << song;
                    break;
                }
            }
        }
    }

    emit downLoadDataChanged( QString() );
    if(m_reply)
    {
        m_reply->deleteLater();
        m_reply = nullptr;
    }
}

void MusicIdentifySongsThread::keyDownLoadFinished(const QByteArray &data)
{
    QJson::Parser parser;
    bool ok;
    QVariant dt = parser.parse(data, &ok);
    if(ok)
    {
        QVariantMap value = dt.toMap();
        if( QDateTime::fromString( value["time"].toString(), "yyyy-MM-dd HH:mm:ss") >
            QDateTime::currentDateTime())
        {
            m_accessKey = value["key"].toString();
            m_accessSecret = value["secret"].toString();
        }
    }
    emit getKeyFinished();
}

QByteArray MusicIdentifySongsThread::hmacSha1(const QByteArray &data, const QByteArray &secretKey)
{
    int blockSize = 64;
    QByteArray newSecretKey = secretKey;
    if(newSecretKey.length() > blockSize)
    {
        newSecretKey = QCryptographicHash::hash(newSecretKey, QCryptographicHash::Sha1);
    }

    QByteArray innerPadding(blockSize, char(0x36));
    QByteArray outerPadding(blockSize, char(0x5c));

    for(int i = 0; i < secretKey.length(); i++)
    {
        innerPadding[i] = innerPadding[i] ^ secretKey.at(i);
        outerPadding[i] = outerPadding[i] ^ secretKey.at(i);
    }

    QByteArray total = outerPadding;
    QByteArray part = innerPadding;
    part.append(data);
    total.append(QCryptographicHash::hash(part, QCryptographicHash::Sha1));
    QByteArray hashed = QCryptographicHash::hash(total, QCryptographicHash::Sha1);
    return hashed;
}
<|endoftext|>


<commit_before>//
// Copyright 2017 Shivansh Rai
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
// SUCH DAMAGE.
//
// $FreeBSD$

#include <array>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <sys/stat.h>
#include <sys/wait.h>
#include "generate_test.h"

void
add_known_testcase(string option,
                   string utility,
                   string output,
                   ofstream& test_script)
{
  string testcase_name;

  // Add testcase name.
  test_script << "atf_test_case ";
  testcase_name = option;
  testcase_name.append("_flag");
  test_script << testcase_name + "\n";

  // Absence of the string "usage:" in `output` denotes
  // that the `option` is a known one and EXIT_SUCCESS
  // will be encountered when it is executed.

  // Add testcase description.
  test_script << testcase_name
               + "_head()\n{\n\tatf_set \"descr\" "
               + "\"Verify the behavior for the option '-"
               + option + "'\"" + "\n}\n\n";

  // Add body of the testcase.
  test_script << testcase_name
               + "_body()\n{\n\tatf_check -s exit:0 ";

  // Match the usage output if generated.
  if (!output.empty())
    test_script << "-o match:\'" + output + "\' ";

  test_script << utility + " -" + option + "\n}\n";
}

void
add_unknown_testcase(string option,
                     string utility,
                     string output,
                     ofstream& test_script)
{
  test_script << "\n\tatf_check -s exit:1 -e inline:\"$usage_output\" "
               + utility + " -" + option;
}

pair<string, int>
exec(const char* cmd)
{
  array<char, 128> buffer;
  string usage_output;
  FILE* pipe = popen(cmd, "r");
  if (!pipe) throw runtime_error("popen() failed!");
  try {
    while (!feof(pipe)) {
        if (fgets(buffer.data(), 128, pipe) != NULL)
            usage_output += buffer.data();
    }
  }
  catch(...) {
    pclose(pipe);
    throw;
  }

  return make_pair<string, int>
         ((string)usage_output, WEXITSTATUS(pclose(pipe)));
}

void
generate_test()
{
  list<utils::opt_rel*> ident_opt_list;
  string test_file;
  string testcase_list;
  string command;
  struct stat buffer;
  ofstream test_fstream;
  ifstream license_fstream;
  pair<string, int> output;

  utils::opt_def f_opts;
  ident_opt_list = f_opts.check_opts();
  test_file = f_opts.utility + "_test.sh";

  // Check if the test file exists. In case it does, stop execution.
  if (stat (test_file.c_str(), &buffer) == 0) {
    cout << "Skipping: Test file already exists" << endl;
    return;
  }

  test_fstream.open(test_file, ios::out);
  license_fstream.open("license", ios::in);

  test_fstream << license_fstream.rdbuf();
  license_fstream.close();

  // If a known option was encountered (i.e. `ident_opt_list` is
  // not empty), produce a testcase to check the validity of the
  // result of that option. If no known option was encountered,
  // produce testcases to verify the correct usage message
  // when using the valid options incorrectly.

  // Add testcases for the options whose usage is unknown.
  if (!f_opts.opt_list.empty()) {
    // Add the "$usage_output" environment variable.
    command = f_opts.utility + " -" + f_opts.opt_list.at(1) + " 2>&1";
    output = exec(command.c_str());
    if (!output.first.empty() && output.second)
      test_fstream << "usage_output=\'" + output.first + "\'\n\n";

    testcase_list.append("\tatf_add_test_case invalid_usage\n");
    test_fstream << "atf_test_case invalid_usage\ninvalid_usage_head()\n{\n\tatf_set \"descr\" \"Verify that the accepted options produce a valid error message in case of an invalid usage\"\n}\n\ninvalid_usage_body()\n{";

    // If invocation of utility under test without any option
    // fails, we add a relevant test under "invalid_usage" testcase.
    command = f_opts.utility + " 2>&1";
    output = exec(command.c_str());

    if (!output.first.empty() && output.second)
      test_fstream << "\n\tatf_check -s exit:1 -e inline:\"$usage_output\" "
                    + f_opts.utility;

    // Execute the utility with options and add tests accordingly.
    for (int i = 0; i < f_opts.opt_list.length(); i++) {
      command = f_opts.utility + " -" + f_opts.opt_list.at(i) + " 2>&1";
      output = exec(command.c_str());

      if (output.second) {
        // Error is generated.
        add_unknown_testcase(string(1, f_opts.opt_list.at(i)),
                             f_opts.utility, output.first, test_fstream);
      }
      else {
        // Output is generated.
        add_known_testcase(string(1, f_opts.opt_list.at(i)),
                           f_opts.utility, output.first, test_fstream);
      }
    }
    test_fstream << "\n}\n\n";
  }

  // Add testcases for known options.
  if (!ident_opt_list.empty()) {
    for (auto i = ident_opt_list.begin(); i != ident_opt_list.end(); i++) {
      command = f_opts.utility + " -" + (*i)->value + " 2>&1";
      output = exec(command.c_str());
      if (output.first.find("usage:") == string::npos) {
        add_known_testcase((*i)->value, f_opts.utility,
                           output.first, test_fstream);
        testcase_list.append("\tatf_add_test_case " + (*i)->value + "_flag\n");
      }
      else {
        // We failed to guess the correct usage.
        add_unknown_testcase((*i)->value, f_opts.utility, output.first, test_fstream);
      }
    }
  }

  test_fstream << "atf_init_test_cases()\n{\n" + testcase_list + "}\n";
  test_fstream.close();
}

int
main()
{
  generate_test();
  return 0;
}
<commit_msg>[WIP]: Add support for utilities which don't accept any options<commit_after>//
// Copyright 2017 Shivansh Rai
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
// SUCH DAMAGE.
//
// $FreeBSD$

#include <array>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <sys/stat.h>
#include <sys/wait.h>
#include "generate_test.h"

void
add_known_testcase(string option,
                   string utility,
                   string descr,
                   string output,
                   ofstream& test_script)
{
  string testcase_name;

  // Add testcase name.
  test_script << "atf_test_case ";
  testcase_name = option;
  testcase_name.append("_flag");
  test_script << testcase_name + "\n";

  // Add testcase description.
  test_script << testcase_name
               + "_head()\n{\n\tatf_set \"descr\" "
               + descr + "\n}\n\n";
               // + "\"Verify the behavior for the option '-"
               // + option + "'\"" + "\n}\n\n";

  // Add body of the testcase.
  test_script << testcase_name
               + "_body()\n{\n\tatf_check -s exit:0 ";

  // Match the usage output if generated.
  if (!output.empty())
    test_script << "-o match:\'" + output + "\' ";

  test_script << utility;

  if (!option.empty())
    test_script << " -" + option;

  test_script << "\n}\n";
}

void
add_unknown_testcase(string option,
                     string utility,
                     string output,
                     ofstream& test_script)
{
  test_script << "\n\tatf_check -s exit:1 -e inline:\"$usage_output\" "
               + utility + " -" + option;
}

pair<string, int>
exec(const char* cmd)
{
  array<char, 128> buffer;
  string usage_output;
  FILE* pipe = popen(cmd, "r");
  if (!pipe) throw runtime_error("popen() failed!");
  try {
    while (!feof(pipe)) {
        if (fgets(buffer.data(), 128, pipe) != NULL)
            usage_output += buffer.data();
    }
  }
  catch(...) {
    pclose(pipe);
    throw;
  }

  return make_pair<string, int>
         ((string)usage_output, WEXITSTATUS(pclose(pipe)));
}

void
generate_test()
{
  list<utils::opt_rel*> ident_opt_list;
  string test_file;
  string testcase_list;
  string command;
  string descr;
  struct stat buffer;
  ofstream test_fstream;
  ifstream license_fstream;
  pair<string, int> output;

  utils::opt_def f_opts;
  ident_opt_list = f_opts.check_opts();
  test_file = f_opts.utility + "_test.sh";

  // Check if the test file exists. In case it does, stop execution.
  if (stat (test_file.c_str(), &buffer) == 0) {
    cout << "Skipping: Test file already exists" << endl;
    return;
  }

  test_fstream.open(test_file, ios::out);
  license_fstream.open("license", ios::in);

  test_fstream << license_fstream.rdbuf();
  license_fstream.close();

  // If invocation of utility under test without any option
  // fails, we add a relevant test under "invalid_usage" testcase.
  command = f_opts.utility + " 2>&1";
  output = exec(command.c_str());

  if (output.second) {
    if (!output.first.empty()) {
      // We expect a usage message to be generated in this case.
      test_fstream << "\n\tatf_check -s exit:1 -e inline:\"$usage_output\" " + f_opts.utility;
    }
    else
      test_fstream << "\n\tatf_check -s exit:1 -e empty " + f_opts.utility;
  }
  else {
    if (!output.first.empty()) {
      descr = "Verify that " + f_opts.utility + " executes successfully and "
            + "produces a valid output when run without any options";
      add_known_testcase("", f_opts.utility, descr, output.first, test_fstream);
    }
    else {
      descr = "Verify that " + f_opts.utility + " executes successfully and "
            + "silently when run without any options";
      add_known_testcase("", f_opts.utility, descr, output.first, test_fstream);
    }
  }

  // If a known option was encountered (i.e. `ident_opt_list` is
  // not empty), produce a testcase to check the validity of the
  // result of that option. If no known option was encountered,
  // produce testcases to verify the correct usage message
  // when using the valid options incorrectly.

  // Add testcases for the options whose usage is unknown.
  if (!f_opts.opt_list.empty()) {
    // Add the "$usage_output" environment variable.
    command = f_opts.utility + " -" + f_opts.opt_list[1] + " 2>&1";
    output = exec(command.c_str());
    if (!output.first.empty() && output.second)
      test_fstream << "usage_output=\'" + output.first + "\'\n\n";

    testcase_list.append("\tatf_add_test_case invalid_usage\n");
    test_fstream << "atf_test_case invalid_usage\ninvalid_usage_head()\n{\n\tatf_set \"descr\" \"Verify that the accepted options produce a valid error message in case of an invalid usage\"\n}\n\ninvalid_usage_body()\n{";

    // Execute the utility with options and add tests accordingly.
    for (int i = 0; i < f_opts.opt_list.length(); i++) {
      command = f_opts.utility + " -" + f_opts.opt_list[i] + " 2>&1";
      output = exec(command.c_str());
      descr = string("Verify the behavior of option \'")
            + f_opts.opt_list[i] + string("\'");

      if (output.second) {
        // Error is generated.
        add_unknown_testcase(string(1, f_opts.opt_list[i]),
                             f_opts.utility, output.first, test_fstream);
      }
      else {
        // Output is generated.
        add_known_testcase(string(1, f_opts.opt_list[i]), descr,
                           f_opts.utility, output.first, test_fstream);
      }
    }
    test_fstream << "\n}\n\n";
  }

  // Add testcases for known options.
  if (!ident_opt_list.empty()) {
    for (auto i = ident_opt_list.begin(); i != ident_opt_list.end(); i++) {
      command = f_opts.utility + " -" + (*i)->value + " 2>&1";
      output = exec(command.c_str());
      if (output.first.find("usage:") == string::npos) {
        add_known_testcase((*i)->value, f_opts.utility, descr,
                           output.first, test_fstream);
        testcase_list.append("\tatf_add_test_case " + (*i)->value + "_flag\n");
      }
      else {
        // We failed to guess the correct usage.
        add_unknown_testcase((*i)->value, f_opts.utility, output.first, test_fstream);
      }
    }
  }

  test_fstream << "atf_init_test_cases()\n{\n" + testcase_list + "}\n";
  test_fstream.close();
}

int
main()
{
  generate_test();
  return 0;
}
<|endoftext|>