<commit_before>#include <QSignalSpy>
#include "shoppinglist-t.h"
#include "shoppinglist.h"
#include <QTableView>

void ShoppingListTest::init()
{
	shoppingList = new ShoppingList;

	//TODO: Since all tests mostly use the same data, populate the shoppingList here.
}

void ShoppingListTest::cleanup()
{
	delete shoppingList;
}

void ShoppingListTest::testNewItem()
{
	QString item_1 = "test_item";
	QString item_2 = "test_item_2";

	double price = 20.0;
	int amount = 10;

	//basic test of add
	QCOMPARE(shoppingList->rowCount(), 0);
	shoppingList->newItem(item_1, price, amount);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), item_1);

	//add another item, verify it is added in a new row
	QCOMPARE(shoppingList->rowCount(), 1);
	shoppingList->newItem(item_2, price, amount);
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), item_1);

	//verify the other elements
	QCOMPARE(shoppingList->data(shoppingList->index(1, 1)).toDouble(), price);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 2)).toInt(), amount);

	//add item with same name, verify it overwrites the former item
	double price_2 = 30.0;
	int amount_2 = 30;
	shoppingList->newItem(item_2, price_2, amount_2);
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 1)).toDouble(), price_2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 2)).toInt(), amount_2);
}

void ShoppingListTest::testSetItemAmount()
{
	int amount = 20;
	shoppingList->newItem("test_item_1", 20.0, amount);
	shoppingList->newItem("test_item_2", 20.0, amount);
	shoppingList->newItem("test_item_3", 20.0, amount);

	QCOMPARE(shoppingList->data(shoppingList->index(2, 2)).toInt(), amount);

	//check that the amount is changed when requested
	shoppingList->setItemAmount("test_item_3", amount*2);
	QCOMPARE(shoppingList->data(shoppingList->index(2, 2)).toInt(), amount*2);
}

void ShoppingListTest::testDeleteItem()
{
	int amount = 20;
	shoppingList->newItem("test_item_1", 20.0, amount);
	shoppingList->newItem("test_item_2", 20.0, amount);
	shoppingList->newItem("test_item_3", 20.0, amount);

	QCOMPARE(shoppingList->rowCount(), 3);

	//check that correct row is deleted when using the item name
	shoppingList->deleteItem("test_item_2");
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), QString("test_item_1"));
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), QString("test_item_3"));

	//check that the correct row is deleted when using the row number
	shoppingList->deleteItem(shoppingList->index(0, 0));
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), QString("test_item_3"));
}

void ShoppingListTest::testWipeList()
{
	int amount = 20;
	shoppingList->newItem("test_item_1", 20.0, amount);
	shoppingList->newItem("test_item_2", 20.0, amount);
	shoppingList->newItem("test_item_3", 20.0, amount);
	QCOMPARE(shoppingList->rowCount(), 3);

	//verify that list is wiped
	shoppingList->wipeList();
	QCOMPARE(shoppingList->rowCount(), 0);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)), QVariant(QVariant::Invalid));
	shoppingList->wipeList();
	QCOMPARE(shoppingList->rowCount(), 0);
}

void ShoppingListTest::testDeleteLastItem()
{
	int amount = 20;
	shoppingList->newItem("test_item_1", 20.0, amount);
	shoppingList->newItem("test_item_2", 20.0, amount);
	shoppingList->newItem("test_item_3", 20.0, amount);
	QCOMPARE(shoppingList->rowCount(), 3);

	//verify that last item was deleted
	shoppingList->deleteLastAddedItem();
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), QString("test_item_1"));
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), QString("test_item_2"));
}

void ShoppingListTest::testGetTotalAmount()
{
	QCOMPARE(shoppingList->getTotalAmount(), 0.0);
	int amount = 20;
	double price = 20;
	shoppingList->newItem("test_item_1", price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), price*amount*1.0);
	shoppingList->newItem("test_item_2", price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), 2*price*amount*1.0);
	shoppingList->newItem("test_item_3", price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), 3*price*amount*1.0);
}

QTEST_MAIN(ShoppingListTest)
<commit_msg>Move testcase data to init()<commit_after>#include <QSignalSpy>
#include "shoppinglist-t.h"
#include "shoppinglist.h"
#include <QTableView>

const QString ITEM_1 = "test_item_1";
const QString ITEM_2 = "test_item_2";
const QString ITEM_3 = "test_item_3";
const double PRICE = 20.0;
const int AMOUNT = 20;

void ShoppingListTest::init()
{
	shoppingList = new ShoppingList;

	//add test data
	shoppingList->newItem(ITEM_1, PRICE, AMOUNT);
	shoppingList->newItem(ITEM_2, PRICE, AMOUNT);
	shoppingList->newItem(ITEM_3, PRICE, AMOUNT);
}

void ShoppingListTest::cleanup()
{
	delete shoppingList;
}

void ShoppingListTest::testNewItem()
{
	//start from scratch (ignore test data created in init()
	delete shoppingList;
	shoppingList = new ShoppingList;

	QString item_1 = "test_item";
	QString item_2 = "test_item_2";

	double price = 20.0;
	int amount = 10;

	//basic test of add
	QCOMPARE(shoppingList->rowCount(), 0);
	shoppingList->newItem(item_1, price, amount);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), item_1);

	//add another item, verify it is added in a new row
	QCOMPARE(shoppingList->rowCount(), 1);
	shoppingList->newItem(item_2, price, amount);
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), item_1);

	//verify the other elements
	QCOMPARE(shoppingList->data(shoppingList->index(1, 1)).toDouble(), price);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 2)).toInt(), amount);

	//add item with same name, verify it overwrites the former item
	double price_2 = 30.0;
	int amount_2 = 30;
	shoppingList->newItem(item_2, price_2, amount_2);
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 1)).toDouble(), price_2);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 2)).toInt(), amount_2);
}

void ShoppingListTest::testSetItemAmount()
{
	QCOMPARE(shoppingList->data(shoppingList->index(2, 2)).toInt(), AMOUNT);

	//check that the amount is changed when requested
	shoppingList->setItemAmount(ITEM_3, AMOUNT*2);
	QCOMPARE(shoppingList->data(shoppingList->index(2, 2)).toInt(), AMOUNT*2);
}

void ShoppingListTest::testDeleteItem()
{
	QCOMPARE(shoppingList->rowCount(), 3);

	//check that correct row is deleted when using the item name
	shoppingList->deleteItem(ITEM_2);
	QCOMPARE(shoppingList->rowCount(), 2);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), ITEM_1);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), ITEM_3);

	//check that the correct row is deleted when using the row number
	shoppingList->deleteItem(shoppingList->index(0, 0));
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), ITEM_3);
}

void ShoppingListTest::testWipeList()
{
	QCOMPARE(shoppingList->rowCount(), 3);

	//verify that list is wiped
	shoppingList->wipeList();
	QCOMPARE(shoppingList->rowCount(), 0);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)), QVariant(QVariant::Invalid));
	shoppingList->wipeList();
	QCOMPARE(shoppingList->rowCount(), 0);
}

void ShoppingListTest::testDeleteLastItem()
{
	shoppingList->newItem("something_different", 20, 20);
	QCOMPARE(shoppingList->rowCount(), 4);

	//verify that last item was deleted
	shoppingList->deleteLastAddedItem();
	QCOMPARE(shoppingList->rowCount(), 3);
	QCOMPARE(shoppingList->data(shoppingList->index(0, 0)).toString(), ITEM_1);
	QCOMPARE(shoppingList->data(shoppingList->index(1, 0)).toString(), ITEM_2);
	QCOMPARE(shoppingList->data(shoppingList->index(2, 0)).toString(), ITEM_3);
}

void ShoppingListTest::testGetTotalAmount()
{
	shoppingList->wipeList();
	QCOMPARE(shoppingList->rowCount(), 0);

	int amount = 20;
	double price = 20;
	shoppingList->newItem(ITEM_1, price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), price*amount*1.0);
	shoppingList->newItem(ITEM_2, price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), 2*price*amount*1.0);
	shoppingList->newItem(ITEM_3, price, amount);
	QCOMPARE(shoppingList->getTotalAmount(), 3*price*amount*1.0);
}

QTEST_MAIN(ShoppingListTest)
<|endoftext|>


<commit_before>#include "mainwindow.h"
#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MainWindow w;
    w.show();

    return a.exec();
}
<commit_msg>comment<commit_after>#include "mainwindow.h"
#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MainWindow w;
    w.show();
//dashov comment
    return a.exec();
}
<|endoftext|>


<commit_before>/*
   This file is part of INDDGO.

   Copyright (C) 2012, Oak Ridge National Laboratory

   This product includes software produced by UT-Battelle, LLC under Contract No.
   DE-AC05-00OR22725 with the Department of Energy.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the New BSD 3-clause software license (LICENSE).

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   LICENSE for more details.

   For more information please contact the INDDGO developers at:
   inddgo-info@googlegroups.com

 */

#include "GraphReader.h"
#include "GraphProperties.h"
#include <ctime>

using namespace std;

void usage(const char *s){
    fprintf(stderr, "Usage: %s filename\n", s);
}

int main(int argc, char **argv){
    //usage check
    if((argc == 1) ||
       ((argc == 2) && (strcmp(argv[1],"-h") == 0)) ||
       ((argc == 2) && (strcmp(argv[1], "--help") == 0))){
        usage(argv[0]);
        exit(-1);
    }

    //check if too many args?

    Graph::Graph *g;

    clock_t begin, end;

    Graph::GraphProperties prop;
    Graph::GraphReader ngr;

    //create the graph object
    g = new Graph::Graph();

    //read the graph from the filename, assume it is an edgelist
    ngr.read_graph(g, argv[1], "Edge", false);
    printf("Read %d vertices and %d edges\n", g->get_num_nodes(), g->get_num_edges());

    printf("Simplifying graph\n");
    begin = clock();
    prop.make_simple(g);    //remove self loops and duplicate edges
    end = clock();
    printf("Time: %f\n", double(end - begin) / CLOCKS_PER_SEC);

    //compute shortest paths from source node
    begin = clock();
    //for each node run it
    int source = 0;
    vector<int> onePaths;

    printf("\nRunning Dijsktra for source %d\n",source);
    prop.paths_dijkstra_single(g, onePaths, source);
    end = clock();
    printf("Alg Time (single): %f\n", double(end - begin) / CLOCKS_PER_SEC);

    //compute all pairs shortest paths
    begin = clock();
    vector< vector<int> > allPaths;

    printf("\nRunning Dijsktra for all pairs \n");
    prop.paths_dijkstra_all(g,allPaths);
    end = clock();
    printf("Alg Time (all): %f\n", double(end - begin) / CLOCKS_PER_SEC);
} // main

<commit_msg>minor edits<commit_after>/*
   This file is part of INDDGO.

   Copyright (C) 2012, Oak Ridge National Laboratory

   This product includes software produced by UT-Battelle, LLC under Contract No.
   DE-AC05-00OR22725 with the Department of Energy.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the New BSD 3-clause software license (LICENSE).

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   LICENSE for more details.

   For more information please contact the INDDGO developers at:
   inddgo-info@googlegroups.com

 */

#include "GraphReader.h"
#include "GraphProperties.h"
#include <ctime>

using namespace std;

void usage(const char *s){
    fprintf(stderr, "Usage: %s filename\n", s);
}

int main(int argc, char **argv){
    //usage check
    if((argc == 1) ||
       ((argc == 2) && (strcmp(argv[1],"-h") == 0)) ||
       ((argc == 2) && (strcmp(argv[1], "--help") == 0))){
        usage(argv[0]);
        exit(-1);
    }

    //check if too many args?

    Graph::Graph *g;

    clock_t begin, end;

    Graph::GraphProperties prop;
    Graph::GraphReader ngr;

    //create the graph object
    g = new Graph::Graph();

    //read the graph from the filename, assume it is an edgelist
    ngr.read_graph(g, argv[1], "Edge", false);
    //ngr.read_graph(g, argv[1], "ADJMATRIX", false);

    printf("Read %d vertices and %d edges\n", g->get_num_nodes(), g->get_num_edges());

    printf("Simplifying graph\n");
    begin = clock();
    prop.make_simple(g);    //remove self loops and duplicate edges
    end = clock();
    printf("Time: %f\n", double(end - begin) / CLOCKS_PER_SEC);

    //compute shortest paths from source node
    begin = clock();
    //for each node run it
    int source = 0;
    vector<int> onePaths;

    printf("\nRunning Dijsktra for source %d\n",source);
    prop.paths_dijkstra_single(g, onePaths, source);
    end = clock();
    printf("Alg Time (single): %f\n", double(end - begin) / CLOCKS_PER_SEC);

    //compute all pairs shortest paths
    begin = clock();
    vector< vector<int> > allPaths;

    printf("\nRunning Dijsktra for all pairs \n");
    prop.paths_dijkstra_all(g,allPaths);
    end = clock();
    printf("Alg Time (all): %f\n", double(end - begin) / CLOCKS_PER_SEC);

} // main

<|endoftext|>


<commit_before>//
// keyvi - A key value store.
//
// Copyright 2015 Hendrik Muhs<hendrik.muhs@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

/*
 * keyvicompiler.cpp
 *
 *  Created on: May 13, 2014
 *      Author: hendrik
 */

#include <boost/program_options.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range.hpp>
#include "dictionary/dictionary_compiler.h"
#include "dictionary/fsa/internal/sparse_array_persistence.h"
#include "dictionary/fsa/internal/int_value_store.h"
#include "dictionary/fsa/internal/string_value_store.h"
#include "dictionary/fsa/internal/json_value_store.h"

typedef keyvi::dictionary::fsa::internal::IValueStoreWriter::vs_param_t vs_param_t;

void callback (size_t added, size_t overall, void*) {
  std::cout << "Processed " << added << "/" << overall << "(" << ((100 * added) / overall) <<  "%)." << std::endl;
}

template<typename CompilerType, typename ValueType>
void compile_multiple(CompilerType& compiler, std::function<std::pair<std::string, ValueType>(std::string)> parser,
                      std::vector<std::string>& inputs)
{
  boost::iostreams::filtering_istream input_stream;
  std::string line;

  for (auto input_as_string : inputs) {
    auto input = boost::filesystem::path(input_as_string);

    if(boost::filesystem::is_directory(input)) {
      int files_added = 0;
      for(auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(input), {})) {
        if (entry.path().extension() == ".gz") {
          input_stream.push(boost::iostreams::gzip_decompressor());
        }

        boost::iostreams::file_source file(entry.path().string(), std::ios_base::in | std::ios_base::binary);
        input_stream.push(file);
        ++files_added;
        while (std::getline(input_stream, line)) {

          auto parse_result = parser(line);
          if (parse_result.first.size() == 0) {
            continue;
          }

          compiler.Add(parse_result.first, parse_result.second);
        }
        input_stream.reset();
      }

    } else {
      if (input.extension() == ".gz"){
        input_stream.push(boost::iostreams::gzip_decompressor());
      }

      boost::iostreams::file_source file(input.string(), std::ios_base::in | std::ios_base::binary);

      input_stream.push(file);
      while (std::getline(input_stream, line)) {
        auto parse_result = parser(line);
        compiler.Add(parse_result.first, parse_result.second);
      }
      input_stream.reset();
    }
  }
}

template<typename CompilerType>
void finalize_compile(CompilerType& compiler, std::string& output,
                      size_t partition_size = 0, const std::string& manifest = "") {
  if (partition_size == 0) {
    std::ofstream out_stream(output, std::ios::binary);
    compiler.Compile(callback);
    try {
      compiler.SetManifestFromString(manifest);
    } catch(boost::property_tree::json_parser::json_parser_error const& error) {
      std::cout << "Failed to set manifest: " << manifest << std::endl;
      std::cout << error.what() << std::endl;
    }
    compiler.Write(out_stream);
    out_stream.close();
  } else {
    std::string output_part_zero = output + ".0";
    int partition_number = 1;
    std::ofstream out_stream(output_part_zero, std::ios::binary);

    while (compiler.CompileNext(partition_size, out_stream, 2, callback)) {
      std::cout << "Finalize partition " << partition_number << std::endl;

      out_stream.close();
      out_stream.open(output + "." + std::to_string(partition_number),
                      std::ios::binary);
      ++partition_number;
    }

    out_stream.close();
  }
}

template<class BucketT = uint32_t>
void compile_integer(std::vector<std::string>& input, std::string& output, size_t memory_limit,
             size_t partition_size = 0, const std::string& manifest = "") {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::IntValueStoreWithInnerWeights> compiler(
      memory_limit);

  std::function<std::pair<std::string, uint32_t>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');

    if (tab == std::string::npos)
      return std::pair<std::string, uint32_t>();

    std::string key = line.substr(0, tab);
    std::string value_as_string = line.substr(tab + 1);
    uint32_t value;

    try {
      value = boost::lexical_cast<uint32_t>(value_as_string);
    } catch (boost::bad_lexical_cast const&) {
      std::cout << "Error: value was not valid: " << line << std::endl;
      return std::pair<std::string, uint32_t>();
    }
      return std::pair<std::string, uint32_t>(key, value);
  };
  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_strings(std::vector<std::string>& input, std::string& output,
                     size_t memory_limit, size_t partition_size = 0, const std::string& manifest = "") {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::StringValueStore> compiler(
          memory_limit);

  std::function<std::pair<std::string, std::string>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');
    if (tab == std::string::npos)
      return std::pair<std::string, std::string>();

    std::string key = line.substr(0, tab);
    std::string value = line.substr(tab + 1);

    return std::pair<std::string, std::string>(key, value);
  };

  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_key_only(std::vector<std::string>& input, std::string& output,
                      size_t memory_limit, size_t partition_size = 0, const std::string& manifest = "") {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>> compiler(
      memory_limit);

  std::function<std::pair<std::string, uint32_t>(std::string)> parser = [] (std::string line) {

    std::string key = line;
    size_t tab = line.find('\t');

    if (tab != std::string::npos) {
      key = line.substr(0, tab);
    }

    return std::pair<std::string, uint32_t>(key, 0);
  };

  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_json(std::vector<std::string>& input, std::string& output, size_t memory_limit,
                  size_t partition_size = 0, const std::string& manifest = "") {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::JsonValueStore> compiler(
          memory_limit);

  std::function<std::pair<std::string, std::string>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');
    if (tab == std::string::npos)
      return std::pair<std::string, std::string>();

    std::string key = line.substr(0, tab);
    std::string value = line.substr(tab + 1);

    return std::pair<std::string, std::string>(key, value);
  };

  compile_multiple(compiler, parser, input);

  //input_stream.
  finalize_compile(compiler, output, partition_size, manifest);
}

vs_param_t extract_value_store_parameters(
    const boost::program_options::variables_map& vm) {
  vs_param_t ret;
  for (auto& kv : vm) {
    if (kv.first[0] == 'V') ret[kv.first] = kv.second;
  }
  return ret;
}

int main(int argc, char** argv) {
  std::vector<std::string> input_files;
  std::string output_file;

  boost::program_options::options_description description(
      "keyvi compiler options:");

  description.add_options()("help,h", "Display this help message")(
      "version,v", "Display the version number");

  description.add_options()("input-file,i",
                            boost::program_options::value<std::vector<std::string>>(),
                            "input file");
  description.add_options()("output-file,o",
                            boost::program_options::value<std::string>(),
                            "output file");
  description.add_options()("memory-limit,m",
                            boost::program_options::value<size_t>(),
                            "amount of main memory to use");
  description.add_options()(
      "dictionary-type,d",
      boost::program_options::value<std::string>()->default_value("integer"),
      "type of dictionary (integer (default), string, key-only, json)");
  description.add_options()("partition-size,p",
                            boost::program_options::value<size_t>(),
                            "create partitions with a maximum size");
  description.add_options()("compact,c", "Compact Mode");

  description.add_options()(
        "manifest",
        boost::program_options::value<std::string>()->default_value(""),
        "manifest to be embedded");

  // Declare which options are positional
  boost::program_options::positional_options_description p;
  p.add("input-file", -1);

  boost::program_options::variables_map vm;
  boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(
          description).run(),
      vm);
  boost::program_options::notify(vm);

  // parse positional options
  boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(
          description).positional(p).run(),
      vm);
  boost::program_options::notify(vm);
  if (vm.count("help")) {
    std::cout << description;
    return 0;
  }

  size_t memory_limit = 1073741824;
  if (vm.count("memory-limit")) {
    memory_limit = vm["memory-limit"].as<size_t>();
  }

  bool compact = false;
  if (vm.count("compact")) {
      compact = true;
  }

  size_t partition_size = 0;
  if (vm.count("partition-size")) {
    partition_size = vm["partition-size"].as<size_t>();
  }

  std::string manifest = vm["manifest"].as<std::string>();
  std::cout << manifest << std::endl;

  vs_param_t value_store_params = extract_value_store_parameters(vm);

  if (vm.count("input-file") && vm.count("output-file")) {
    input_files = vm["input-file"].as<std::vector<std::string>>();
    output_file = vm["output-file"].as<std::string>();

    std::string dictionary_type = vm["dictionary-type"].as<std::string>();
    if (dictionary_type == "integer") {
      if (compact){
        compile_integer<uint16_t>(input_files, output_file, memory_limit,
                                  partition_size, manifest, value_store_params);
      } else {
        compile_integer(input_files, output_file, memory_limit, partition_size,
                        manifest, value_store_params);
      }
    } else if (dictionary_type == "string") {
      if (compact){
        compile_strings<uint16_t>(input_files, output_file, memory_limit,
                                  partition_size, manifest, value_store_params);
      } else {
        compile_strings(input_files, output_file, memory_limit, partition_size,
                        manifest, value_store_params);
      }
    } else if (dictionary_type == "key-only") {
      if (compact){
        compile_key_only<uint16_t>(input_files, output_file, memory_limit,
                                   partition_size, manifest, value_store_params);
      } else {
        compile_key_only(input_files, output_file, memory_limit, partition_size,
                         manifest, value_store_params);
      }
    } else if (dictionary_type == "json") {
      if (compact){
        compile_json<uint16_t>(input_files, output_file, memory_limit,
                               partition_size, manifest, value_store_params);
      } else {
        compile_json(input_files, output_file, memory_limit, partition_size,
                     manifest, value_store_params);
      }
    } else {
      std::cout << "ERROR: unknown dictionary type." << std::endl << std::endl;
      std::cout << description;
      return 1;
    }
  } else {
    std::cout << "ERROR: arguments wrong or missing." << std::endl << std::endl;
    std::cout << description;
    return 1;
  }

  return 0;
}
<commit_msg>Linked it?<commit_after>//
// keyvi - A key value store.
//
// Copyright 2015 Hendrik Muhs<hendrik.muhs@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

/*
 * keyvicompiler.cpp
 *
 *  Created on: May 13, 2014
 *      Author: hendrik
 */

#include <boost/program_options.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range.hpp>
#include "dictionary/dictionary_compiler.h"
#include "dictionary/fsa/internal/sparse_array_persistence.h"
#include "dictionary/fsa/internal/int_value_store.h"
#include "dictionary/fsa/internal/string_value_store.h"
#include "dictionary/fsa/internal/json_value_store.h"

typedef keyvi::dictionary::fsa::internal::IValueStoreWriter::vs_param_t vs_param_t;

void callback (size_t added, size_t overall, void*) {
  std::cout << "Processed " << added << "/" << overall << "(" << ((100 * added) / overall) <<  "%)." << std::endl;
}

template<typename CompilerType, typename ValueType>
void compile_multiple(CompilerType& compiler, std::function<std::pair<std::string, ValueType>(std::string)> parser,
                      std::vector<std::string>& inputs)
{
  boost::iostreams::filtering_istream input_stream;
  std::string line;

  for (auto input_as_string : inputs) {
    auto input = boost::filesystem::path(input_as_string);

    if(boost::filesystem::is_directory(input)) {
      int files_added = 0;
      for(auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(input), {})) {
        if (entry.path().extension() == ".gz") {
          input_stream.push(boost::iostreams::gzip_decompressor());
        }

        boost::iostreams::file_source file(entry.path().string(), std::ios_base::in | std::ios_base::binary);
        input_stream.push(file);
        ++files_added;
        while (std::getline(input_stream, line)) {

          auto parse_result = parser(line);
          if (parse_result.first.size() == 0) {
            continue;
          }

          compiler.Add(parse_result.first, parse_result.second);
        }
        input_stream.reset();
      }

    } else {
      if (input.extension() == ".gz"){
        input_stream.push(boost::iostreams::gzip_decompressor());
      }

      boost::iostreams::file_source file(input.string(), std::ios_base::in | std::ios_base::binary);

      input_stream.push(file);
      while (std::getline(input_stream, line)) {
        auto parse_result = parser(line);
        compiler.Add(parse_result.first, parse_result.second);
      }
      input_stream.reset();
    }
  }
}

template<typename CompilerType>
void finalize_compile(CompilerType& compiler, std::string& output,
                      size_t partition_size = 0, const std::string& manifest = "") {
  if (partition_size == 0) {
    std::ofstream out_stream(output, std::ios::binary);
    compiler.Compile(callback);
    try {
      compiler.SetManifestFromString(manifest);
    } catch(boost::property_tree::json_parser::json_parser_error const& error) {
      std::cout << "Failed to set manifest: " << manifest << std::endl;
      std::cout << error.what() << std::endl;
    }
    compiler.Write(out_stream);
    out_stream.close();
  } else {
    std::string output_part_zero = output + ".0";
    int partition_number = 1;
    std::ofstream out_stream(output_part_zero, std::ios::binary);

    while (compiler.CompileNext(partition_size, out_stream, 2, callback)) {
      std::cout << "Finalize partition " << partition_number << std::endl;

      out_stream.close();
      out_stream.open(output + "." + std::to_string(partition_number),
                      std::ios::binary);
      ++partition_number;
    }

    out_stream.close();
  }
}

template<class BucketT = uint32_t>
void compile_integer(std::vector<std::string>& input, std::string& output,
                     size_t memory_limit, size_t partition_size = 0,
                     const std::string& manifest = "",
                     const vs_param_t& value_store_params = vs_param_t()) {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::IntValueStoreWithInnerWeights> compiler(
      memory_limit, value_store_params);

  std::function<std::pair<std::string, uint32_t>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');

    if (tab == std::string::npos)
      return std::pair<std::string, uint32_t>();

    std::string key = line.substr(0, tab);
    std::string value_as_string = line.substr(tab + 1);
    uint32_t value;

    try {
      value = boost::lexical_cast<uint32_t>(value_as_string);
    } catch (boost::bad_lexical_cast const&) {
      std::cout << "Error: value was not valid: " << line << std::endl;
      return std::pair<std::string, uint32_t>();
    }
      return std::pair<std::string, uint32_t>(key, value);
  };
  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_strings(std::vector<std::string>& input, std::string& output,
                     size_t memory_limit, size_t partition_size = 0,
                     const std::string& manifest = "",
                     const vs_param_t& value_store_params = vs_param_t()) {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::StringValueStore> compiler(
          memory_limit, value_store_params);

  std::function<std::pair<std::string, std::string>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');
    if (tab == std::string::npos)
      return std::pair<std::string, std::string>();

    std::string key = line.substr(0, tab);
    std::string value = line.substr(tab + 1);

    return std::pair<std::string, std::string>(key, value);
  };

  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_key_only(std::vector<std::string>& input, std::string& output,
                      size_t memory_limit, size_t partition_size = 0,
                      const std::string& manifest = "",
                      const vs_param_t& value_store_params = vs_param_t()) {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>> compiler(
      memory_limit, value_store_params);

  std::function<std::pair<std::string, uint32_t>(std::string)> parser = [] (std::string line) {

    std::string key = line;
    size_t tab = line.find('\t');

    if (tab != std::string::npos) {
      key = line.substr(0, tab);
    }

    return std::pair<std::string, uint32_t>(key, 0);
  };

  compile_multiple(compiler, parser, input);

  finalize_compile(compiler, output, partition_size, manifest);
}

template<class BucketT = uint32_t>
void compile_json(std::vector<std::string>& input, std::string& output,
                  size_t memory_limit, size_t partition_size = 0,
                  const std::string& manifest = "",
                  const vs_param_t& value_store_params = vs_param_t()) {
  keyvi::dictionary::DictionaryCompiler<
      keyvi::dictionary::fsa::internal::SparseArrayPersistence<BucketT>,
      keyvi::dictionary::fsa::internal::JsonValueStore> compiler(
          memory_limit, value_store_params);

  std::function<std::pair<std::string, std::string>(std::string)> parser = [] (std::string line) {
    size_t tab = line.find('\t');
    if (tab == std::string::npos)
      return std::pair<std::string, std::string>();

    std::string key = line.substr(0, tab);
    std::string value = line.substr(tab + 1);

    return std::pair<std::string, std::string>(key, value);
  };

  compile_multiple(compiler, parser, input);

  //input_stream.
  finalize_compile(compiler, output, partition_size, manifest);
}

vs_param_t extract_value_store_parameters(
    const boost::program_options::variables_map& vm) {
  vs_param_t ret;
  for (auto& kv : vm) {
    if (kv.first[0] == 'V') ret[kv.first] = kv.second.as<std::string>();
  }
  return ret;
}

int main(int argc, char** argv) {
  std::vector<std::string> input_files;
  std::string output_file;

  boost::program_options::options_description description(
      "keyvi compiler options:");

  description.add_options()("help,h", "Display this help message")(
      "version,v", "Display the version number");

  description.add_options()("input-file,i",
                            boost::program_options::value<std::vector<std::string>>(),
                            "input file");
  description.add_options()("output-file,o",
                            boost::program_options::value<std::string>(),
                            "output file");
  description.add_options()("memory-limit,m",
                            boost::program_options::value<size_t>(),
                            "amount of main memory to use");
  description.add_options()(
      "dictionary-type,d",
      boost::program_options::value<std::string>()->default_value("integer"),
      "type of dictionary (integer (default), string, key-only, json)");
  description.add_options()("partition-size,p",
                            boost::program_options::value<size_t>(),
                            "create partitions with a maximum size");
  description.add_options()("compact,c", "Compact Mode");

  description.add_options()(
        "manifest",
        boost::program_options::value<std::string>()->default_value(""),
        "manifest to be embedded");

  // Declare which options are positional
  boost::program_options::positional_options_description p;
  p.add("input-file", -1);

  boost::program_options::variables_map vm;
  boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(
          description).run(),
      vm);
  boost::program_options::notify(vm);

  // parse positional options
  boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(
          description).positional(p).run(),
      vm);
  boost::program_options::notify(vm);
  if (vm.count("help")) {
    std::cout << description;
    return 0;
  }

  size_t memory_limit = 1073741824;
  if (vm.count("memory-limit")) {
    memory_limit = vm["memory-limit"].as<size_t>();
  }

  bool compact = false;
  if (vm.count("compact")) {
      compact = true;
  }

  size_t partition_size = 0;
  if (vm.count("partition-size")) {
    partition_size = vm["partition-size"].as<size_t>();
  }

  std::string manifest = vm["manifest"].as<std::string>();
  std::cout << manifest << std::endl;

  vs_param_t value_store_params = extract_value_store_parameters(vm);

  if (vm.count("input-file") && vm.count("output-file")) {
    input_files = vm["input-file"].as<std::vector<std::string>>();
    output_file = vm["output-file"].as<std::string>();

    std::string dictionary_type = vm["dictionary-type"].as<std::string>();
    if (dictionary_type == "integer") {
      if (compact){
        compile_integer<uint16_t>(input_files, output_file, memory_limit,
                                  partition_size, manifest, value_store_params);
      } else {
        compile_integer(input_files, output_file, memory_limit, partition_size,
                        manifest, value_store_params);
      }
    } else if (dictionary_type == "string") {
      if (compact){
        compile_strings<uint16_t>(input_files, output_file, memory_limit,
                                  partition_size, manifest, value_store_params);
      } else {
        compile_strings(input_files, output_file, memory_limit, partition_size,
                        manifest, value_store_params);
      }
    } else if (dictionary_type == "key-only") {
      if (compact){
        compile_key_only<uint16_t>(input_files, output_file, memory_limit,
                                   partition_size, manifest, value_store_params);
      } else {
        compile_key_only(input_files, output_file, memory_limit, partition_size,
                         manifest, value_store_params);
      }
    } else if (dictionary_type == "json") {
      if (compact){
        compile_json<uint16_t>(input_files, output_file, memory_limit,
                               partition_size, manifest, value_store_params);
      } else {
        compile_json(input_files, output_file, memory_limit, partition_size,
                     manifest, value_store_params);
      }
    } else {
      std::cout << "ERROR: unknown dictionary type." << std::endl << std::endl;
      std::cout << description;
      return 1;
    }
  } else {
    std::cout << "ERROR: arguments wrong or missing." << std::endl << std::endl;
    std::cout << description;
    return 1;
  }

  return 0;
}
<|endoftext|>


<commit_before>/*

Copyright (c) 2003, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include <vector>
#include <iostream>
#include <cctype>
#include <iomanip>
#include <sstream>

#include "zlib.h"

#include "libtorrent/tracker_manager.hpp"
#include "libtorrent/udp_tracker_connection.hpp"
#include "libtorrent/io.hpp"

namespace
{
	enum
	{
		udp_connection_retries = 4,
		udp_announce_retries = 15,
		udp_connect_timeout = 15,
		udp_announce_timeout = 10,
		udp_buffer_size = 2048
	};
}

using namespace boost::posix_time;

namespace libtorrent
{

	udp_tracker_connection::udp_tracker_connection(
		tracker_request const& req
		, std::string const& hostname
		, unsigned short port
		, boost::weak_ptr<request_callback> c
		, const http_settings& stn)
		: tracker_connection(c)
		, m_request_time(second_clock::universal_time())
		, m_request(req)
		, m_transaction_id(0)
		, m_connection_id(0)
		, m_settings(stn)
		, m_attempts(0)
	{
		// TODO: this is a problem. DNS-lookup is blocking!
		// (may block up to 5 seconds)
		address a(hostname.c_str(), port);
		if (has_requester()) requester().m_tracker_address = a;
		m_socket.reset(new socket(socket::udp, false));
		m_socket->connect(a);

		send_udp_connect();
	}

	bool udp_tracker_connection::send_finished() const
	{
		using namespace boost::posix_time;

		time_duration d = second_clock::universal_time() - m_request_time;
		return (m_transaction_id != 0
			&& m_connection_id != 0)
			|| d > seconds(m_settings.tracker_timeout);
	}

	bool udp_tracker_connection::tick()
	{
		using namespace boost::posix_time;

		time_duration d = second_clock::universal_time() - m_request_time;
		if (m_connection_id == 0
			&& d > seconds(udp_connect_timeout))
		{
			if (m_attempts >= udp_connection_retries)
			{
				if (has_requester())
					requester().tracker_request_timed_out(m_request);
				return true;
			}
			send_udp_connect();
			return false;
		}

		if (m_connection_id != 0
			&& d > seconds(udp_announce_timeout))
		{
			if (m_attempts >= udp_announce_retries)
			{
				if (has_requester())
					requester().tracker_request_timed_out(m_request);
				return true;
			}

			if (m_request.kind == tracker_request::announce_request)
				send_udp_announce();
			else if (m_request.kind == tracker_request::scrape_request)
				send_udp_scrape();
			return false;
		}

		char buf[udp_buffer_size];
		int ret = m_socket->receive(buf, udp_buffer_size);

		// if there was nothing to receive, return
		if (ret == 0) return false;
		if (ret < 0)
		{
			socket::error_code err = m_socket->last_error();
			if (err == socket::would_block) return false;
			throw network_error(m_socket->last_error());
		}
		if (ret == udp_buffer_size)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, "tracker reply too big");
			return true;
		}

		if (m_connection_id == 0)
		{
			return parse_connect_response(buf, ret);
		}
		else if (m_request.kind == tracker_request::announce_request)
		{
			return parse_announce_response(buf, ret);
		}
		else if (m_request.kind == tracker_request::scrape_request)
		{
			return parse_scrape_response(buf, ret);
		}

		assert(false);
		return false;
	}

	void udp_tracker_connection::send_udp_announce()
	{
		if (m_transaction_id == 0)
			m_transaction_id = rand() | (rand() << 16);

		std::vector<char> buf;
		std::back_insert_iterator<std::vector<char> > out(buf);

		// connection_id
		detail::write_int64(m_connection_id, out);
		// action (announce)
		detail::write_int32(announce, out);
		// transaction_id
		detail::write_int32(m_transaction_id, out);
		// info_hash
		std::copy(m_request.info_hash.begin(), m_request.info_hash.end(), out);
		// peer_id
		std::copy(m_request.id.begin(), m_request.id.end(), out);
		// downloaded
		detail::write_int64(m_request.downloaded, out);
		// left
		detail::write_int64(m_request.left, out);
		// uploaded
		detail::write_int64(m_request.uploaded, out);
		// event
		detail::write_int32(m_request.event, out);
		// ip address
		detail::write_int32(0, out);
		// key
		detail::write_int32(m_request.key, out);
		// num_want
		detail::write_int32(m_request.num_want, out);
		// port
		detail::write_uint16(m_request.listen_port, out);

		m_socket->send(&buf[0], buf.size());
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	void udp_tracker_connection::send_udp_scrape()
	{
		if (m_transaction_id == 0)
			m_transaction_id = rand() | (rand() << 16);

		std::vector<char> buf;
		std::back_insert_iterator<std::vector<char> > out(buf);

		// connection_id
		detail::write_int64(m_connection_id, out);
		// action (scrape)
		detail::write_int32(scrape, out);
		// transaction_id
		detail::write_int32(m_transaction_id, out);
		// info_hash
		std::copy(m_request.info_hash.begin(), m_request.info_hash.end(), out);

		m_socket->send(&buf[0], buf.size());
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	void udp_tracker_connection::send_udp_connect()
	{
		char send_buf[16];
		char* ptr = send_buf;

		if (m_transaction_id == 0)
			m_transaction_id = rand() | (rand() << 16);

		// connection_id
		detail::write_int64(0, ptr);
		// action (connect)
		detail::write_int32(connect, ptr);
		// transaction_id
		detail::write_int32(m_transaction_id, ptr);

		m_socket->send(send_buf, 16);
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	bool udp_tracker_connection::parse_announce_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}

		int action = detail::read_int32(buf);
		int transaction = detail::read_int32(buf);
		if (transaction != m_transaction_id)
		{
			return false;
		}

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(buf, buf + len - 8));
			return true;
		}
		if (action != announce) return false;

		if (len < 20)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 20, ignoring");
#endif
			return false;
		}
		int interval = detail::read_int32(buf);
		int incomplete = detail::read_int32(buf);
		int complete = detail::read_int32(buf);
		int num_peers = (len - 20) / 6;
		if ((len - 20) % 6 != 0)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, "invalid tracker response");
			return true;
		}

		if (!has_requester()) return true;

		std::vector<peer_entry> peer_list;
		for (int i = 0; i < num_peers; ++i)
		{
			peer_entry e;
			std::stringstream s;
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf);
			e.ip = s.str();
			e.port = detail::read_uint16(buf);
			e.id.clear();
			peer_list.push_back(e);
		}

		requester().tracker_response(m_request, peer_list, interval
			, complete, incomplete);
		return true;
	}

	bool udp_tracker_connection::parse_scrape_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}

		int action = detail::read_int32(buf);
		int transaction = detail::read_int32(buf);
		if (transaction != m_transaction_id)
		{
			return false;
		}

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(buf, buf + len - 8));
			return true;
		}
		if (action != scrape) return false;

		if (len < 20)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 20, ignoring");
#endif
			return false;
		}
		int complete = detail::read_int32(buf);
		int completed = detail::read_int32(buf);
		int incomplete = detail::read_int32(buf);

		if (!has_requester()) return true;

		std::vector<peer_entry> peer_list;
		requester().tracker_response(m_request, peer_list, 0
			, complete, incomplete);
		return true;
	}

	bool udp_tracker_connection::parse_connect_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}
		const char* ptr = buf;
		int action = detail::read_int32(ptr);
		int transaction = detail::read_int32(ptr);

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(ptr, buf + len));
			return true;
		}
		if (action != connect) return false;
		if (m_transaction_id != transaction)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with incorrect transaction id, ignoring");
#endif
			return false;
		}

		if (len < 16)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a connection message size < 16, ignoring");
#endif
			return false;
		}
		// reset transaction
		m_transaction_id = 0;
		m_attempts = 0;
		m_connection_id = detail::read_int64(ptr);

		if (m_request.kind == tracker_request::announce_request)
			send_udp_announce();
		else if (m_request.kind == tracker_request::scrape_request)
			send_udp_scrape();

		return false;
	}

}

<commit_msg>*** empty log message ***<commit_after>/*

Copyright (c) 2003, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include <vector>
#include <iostream>
#include <cctype>
#include <iomanip>
#include <sstream>

#include "zlib.h"

#include "libtorrent/tracker_manager.hpp"
#include "libtorrent/udp_tracker_connection.hpp"
#include "libtorrent/io.hpp"

namespace
{
	enum
	{
		udp_connection_retries = 4,
		udp_announce_retries = 15,
		udp_connect_timeout = 15,
		udp_announce_timeout = 10,
		udp_buffer_size = 2048
	};
}

using namespace boost::posix_time;

namespace libtorrent
{

	udp_tracker_connection::udp_tracker_connection(
		tracker_request const& req
		, std::string const& hostname
		, unsigned short port
		, boost::weak_ptr<request_callback> c
		, const http_settings& stn)
		: tracker_connection(c)
		, m_request_time(second_clock::universal_time())
		, m_request(req)
		, m_transaction_id(0)
		, m_connection_id(0)
		, m_settings(stn)
		, m_attempts(0)
	{
		// TODO: this is a problem. DNS-lookup is blocking!
		// (may block up to 5 seconds)
		address a(hostname.c_str(), port);
		if (has_requester()) requester().m_tracker_address = a;
		m_socket.reset(new socket(socket::udp, false));
		m_socket->connect(a);

		send_udp_connect();
	}

	bool udp_tracker_connection::send_finished() const
	{
		using namespace boost::posix_time;

		time_duration d = second_clock::universal_time() - m_request_time;
		return (m_transaction_id != 0
			&& m_connection_id != 0)
			|| d > seconds(m_settings.tracker_timeout);
	}

	bool udp_tracker_connection::tick()
	{
		using namespace boost::posix_time;

		time_duration d = second_clock::universal_time() - m_request_time;
		if (m_connection_id == 0
			&& d > seconds(udp_connect_timeout))
		{
			if (m_attempts >= udp_connection_retries)
			{
				if (has_requester())
					requester().tracker_request_timed_out(m_request);
				return true;
			}
			send_udp_connect();
			return false;
		}

		if (m_connection_id != 0
			&& d > seconds(udp_announce_timeout))
		{
			if (m_attempts >= udp_announce_retries)
			{
				if (has_requester())
					requester().tracker_request_timed_out(m_request);
				return true;
			}

			if (m_request.kind == tracker_request::announce_request)
				send_udp_announce();
			else if (m_request.kind == tracker_request::scrape_request)
				send_udp_scrape();
			return false;
		}

		char buf[udp_buffer_size];
		int ret = m_socket->receive(buf, udp_buffer_size);

		// if there was nothing to receive, return
		if (ret == 0) return false;
		if (ret < 0)
		{
			socket::error_code err = m_socket->last_error();
			if (err == socket::would_block) return false;
			throw network_error(m_socket->last_error());
		}
		if (ret == udp_buffer_size)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, "tracker reply too big");
			return true;
		}

		if (m_connection_id == 0)
		{
			return parse_connect_response(buf, ret);
		}
		else if (m_request.kind == tracker_request::announce_request)
		{
			return parse_announce_response(buf, ret);
		}
		else if (m_request.kind == tracker_request::scrape_request)
		{
			return parse_scrape_response(buf, ret);
		}

		assert(false);
		return false;
	}

	void udp_tracker_connection::send_udp_announce()
	{
		if (m_transaction_id == 0)
			m_transaction_id = rand() ^ (rand() << 16);

		std::vector<char> buf;
		std::back_insert_iterator<std::vector<char> > out(buf);

		// connection_id
		detail::write_int64(m_connection_id, out);
		// action (announce)
		detail::write_int32(announce, out);
		// transaction_id
		detail::write_int32(m_transaction_id, out);
		// info_hash
		std::copy(m_request.info_hash.begin(), m_request.info_hash.end(), out);
		// peer_id
		std::copy(m_request.id.begin(), m_request.id.end(), out);
		// downloaded
		detail::write_int64(m_request.downloaded, out);
		// left
		detail::write_int64(m_request.left, out);
		// uploaded
		detail::write_int64(m_request.uploaded, out);
		// event
		detail::write_int32(m_request.event, out);
		// ip address
		detail::write_int32(0, out);
		// key
		detail::write_int32(m_request.key, out);
		// num_want
		detail::write_int32(m_request.num_want, out);
		// port
		detail::write_uint16(m_request.listen_port, out);

		m_socket->send(&buf[0], buf.size());
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	void udp_tracker_connection::send_udp_scrape()
	{
		if (m_transaction_id == 0)
			m_transaction_id = rand() ^ (rand() << 16);

		std::vector<char> buf;
		std::back_insert_iterator<std::vector<char> > out(buf);

		// connection_id
		detail::write_int64(m_connection_id, out);
		// action (scrape)
		detail::write_int32(scrape, out);
		// transaction_id
		detail::write_int32(m_transaction_id, out);
		// info_hash
		std::copy(m_request.info_hash.begin(), m_request.info_hash.end(), out);

		m_socket->send(&buf[0], buf.size());
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	void udp_tracker_connection::send_udp_connect()
	{
		char send_buf[16];
		char* ptr = send_buf;

		if (m_transaction_id == 0)
			m_transaction_id = rand() ^ (rand() << 16);

		// connection_id
		detail::write_int64(0x41727101980, ptr);
		// action (connect)
		detail::write_int32(connect, ptr);
		// transaction_id
		detail::write_int32(m_transaction_id, ptr);

		m_socket->send(send_buf, 16);
		m_request_time = second_clock::universal_time();
		++m_attempts;
	}

	bool udp_tracker_connection::parse_announce_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}

		int action = detail::read_int32(buf);
		int transaction = detail::read_int32(buf);
		if (transaction != m_transaction_id)
		{
			return false;
		}

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(buf, buf + len - 8));
			return true;
		}
		if (action != announce) return false;

		if (len < 20)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 20, ignoring");
#endif
			return false;
		}
		int interval = detail::read_int32(buf);
		int incomplete = detail::read_int32(buf);
		int complete = detail::read_int32(buf);
		int num_peers = (len - 20) / 6;
		if ((len - 20) % 6 != 0)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, "invalid tracker response");
			return true;
		}

		if (!has_requester()) return true;

		std::vector<peer_entry> peer_list;
		for (int i = 0; i < num_peers; ++i)
		{
			peer_entry e;
			std::stringstream s;
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf) << ".";
			s << (int)detail::read_uint8(buf);
			e.ip = s.str();
			e.port = detail::read_uint16(buf);
			e.id.clear();
			peer_list.push_back(e);
		}

		requester().tracker_response(m_request, peer_list, interval
			, complete, incomplete);
		return true;
	}

	bool udp_tracker_connection::parse_scrape_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}

		int action = detail::read_int32(buf);
		int transaction = detail::read_int32(buf);
		if (transaction != m_transaction_id)
		{
			return false;
		}

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(buf, buf + len - 8));
			return true;
		}
		if (action != scrape) return false;

		if (len < 20)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 20, ignoring");
#endif
			return false;
		}
		int complete = detail::read_int32(buf);
		int completed = detail::read_int32(buf);
		int incomplete = detail::read_int32(buf);

		if (!has_requester()) return true;

		std::vector<peer_entry> peer_list;
		requester().tracker_response(m_request, peer_list, 0
			, complete, incomplete);
		return true;
	}

	bool udp_tracker_connection::parse_connect_response(const char* buf, int len)
	{
		assert(buf != 0);
		assert(len > 0);

		if (len < 8)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with size < 8, ignoring");
#endif
			return false;
		}
		const char* ptr = buf;
		int action = detail::read_int32(ptr);
		int transaction = detail::read_int32(ptr);

		if (action == error)
		{
			if (has_requester())
				requester().tracker_request_error(
					m_request, -1, std::string(ptr, buf + len));
			return true;
		}
		if (action != connect) return false;
		if (m_transaction_id != transaction)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a message with incorrect transaction id, ignoring");
#endif
			return false;
		}

		if (len < 16)
		{
#ifdef TORRENT_VERBOSE_LOGGING
			if (has_requester())
				requester().debug_log("udp_tracker_connection: "
				"got a connection message size < 16, ignoring");
#endif
			return false;
		}
		// reset transaction
		m_transaction_id = 0;
		m_attempts = 0;
		m_connection_id = detail::read_int64(ptr);

		if (m_request.kind == tracker_request::announce_request)
			send_udp_announce();
		else if (m_request.kind == tracker_request::scrape_request)
			send_udp_scrape();

		return false;
	}

}

<|endoftext|>


<commit_before>#ifndef STAN_MATH_PRIM_PROB_MULTI_NORMAL_CHOLESKY_LPDF_HPP
#define STAN_MATH_PRIM_PROB_MULTI_NORMAL_CHOLESKY_LPDF_HPP

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/err.hpp>
#include <stan/math/prim/fun/as_value_column_vector_or_scalar.hpp>
#include <stan/math/prim/fun/constants.hpp>
#include <stan/math/prim/fun/dot_self.hpp>
#include <stan/math/prim/fun/log.hpp>
#include <stan/math/prim/fun/max_size_mvt.hpp>
#include <stan/math/prim/fun/mdivide_left_tri.hpp>
#include <stan/math/prim/fun/size_mvt.hpp>
#include <stan/math/prim/fun/sum.hpp>
#include <stan/math/prim/fun/transpose.hpp>
#include <stan/math/prim/fun/vector_seq_view.hpp>
#include <stan/math/prim/functor/operands_and_partials.hpp>

namespace stan {
namespace math {

/** \ingroup multivar_dists
 * The log of the multivariate normal density for the given y, mu, and
 * a Cholesky factor L of the variance matrix.
 * Sigma = LL', a square, semi-positive definite matrix.
 *
 * Analytic expressions taken from
 * http://qwone.com/~jason/writing/multivariateNormal.pdf
 * written by Jason D. M. Rennie.
 *
 * @param y A scalar vector
 * @param mu The mean vector of the multivariate normal distribution.
 * @param L The Cholesky decomposition of a variance matrix
 * of the multivariate normal distribution
 * @return The log of the multivariate normal density.
 * @throw std::domain_error if LL' is not square, not symmetric,
 * or not semi-positive definite.
 * @tparam T_y Type of scalar.
 * @tparam T_loc Type of location.
 * @tparam T_covar Type of scale.
 */
template <bool propto, typename T_y, typename T_loc, typename T_covar,
          require_all_not_nonscalar_prim_or_rev_kernel_expression_t<
              T_y, T_loc, T_covar>* = nullptr>
return_type_t<T_y, T_loc, T_covar> multi_normal_cholesky_lpdf(
    const T_y& y, const T_loc& mu, const T_covar& L) {
  static const char* function = "multi_normal_cholesky_lpdf";
  using T_covar_elem = typename scalar_type<T_covar>::type;
  using T_return = return_type_t<T_y, T_loc, T_covar>;
  using T_partials_return = partials_return_t<T_y, T_loc, T_covar>;
  using matrix_partials_t
      = Eigen::Matrix<T_partials_return, Eigen::Dynamic, Eigen::Dynamic>;
  using vector_partials_t = Eigen::Matrix<T_partials_return, Eigen::Dynamic, 1>;
  using row_vector_partials_t
      = Eigen::Matrix<T_partials_return, 1, Eigen::Dynamic>;
  using T_y_ref = ref_type_t<T_y>;
  using T_mu_ref = ref_type_t<T_loc>;
  using T_L_ref = ref_type_t<T_covar>;

  check_consistent_sizes_mvt(function, "y", y, "mu", mu);
  size_t number_of_y = size_mvt(y);
  size_t number_of_mu = size_mvt(mu);
  if (number_of_y == 0 || number_of_mu == 0) {
    return 0;
  }

  T_y_ref y_ref = y;
  T_mu_ref mu_ref = mu;
  T_L_ref L_ref = L;
  vector_seq_view<T_y_ref> y_vec(y_ref);
  vector_seq_view<T_mu_ref> mu_vec(mu_ref);
  const size_t size_vec = max_size_mvt(y, mu);

  const int size_y = y_vec[0].size();
  const int size_mu = mu_vec[0].size();
  if (likely(size_vec > 1)) {
    // check size consistency of all random variables y
    for (size_t i = 1, size_mvt_y = size_mvt(y); i < size_mvt_y; i++) {
      check_size_match(function,
                       "Size of one of the vectors of "
                       "the random variable",
                       y_vec[i].size(),
                       "Size of the first vector of the "
                       "random variable",
                       size_y);
    }
    // check size consistency of all means mu
    for (size_t i = 1, size_mvt_mu = size_mvt(mu); i < size_mvt_mu; i++) {
      check_size_match(function,
                       "Size of one of the vectors of "
                       "the location variable",
                       mu_vec[i].size(),
                       "Size of the first vector of the "
                       "location variable",
                       size_mu);
    }
  }

  check_size_match(function, "Size of random variable", size_y,
                   "size of location parameter", size_mu);
  check_size_match(function, "Size of random variable", size_y,
                   "rows of covariance parameter", L.rows());
  check_size_match(function, "Size of random variable", size_y,
                   "columns of covariance parameter", L.cols());

  for (size_t i = 0; i < size_vec; i++) {
    check_finite(function, "Location parameter", mu_vec[i]);
    check_not_nan(function, "Random variable", y_vec[i]);
  }

  if (unlikely(size_y == 0)) {
    return T_return(0);
  }

  operands_and_partials<T_y_ref, T_mu_ref, T_L_ref> ops_partials(y_ref, mu_ref,
                                                                 L_ref);

  T_partials_return logp(0);
  if (include_summand<propto>::value) {
    logp += NEG_LOG_SQRT_TWO_PI * size_y * size_vec;
  }

  const matrix_partials_t inv_L_dbl
      = mdivide_left_tri<Eigen::Lower>(value_of(L_ref));

  if (include_summand<propto, T_y, T_loc, T_covar_elem>::value) {
    for (size_t i = 0; i < size_vec; i++) {
      decltype(auto) y_val = as_value_column_vector_or_scalar(y_vec[i]);
      decltype(auto) mu_val = as_value_column_vector_or_scalar(mu_vec[i]);

      const row_vector_partials_t half
          = (inv_L_dbl.template triangularView<Eigen::Lower>()
             * (y_val - mu_val)
                   .template cast<T_partials_return>())
                .transpose();
      const vector_partials_t scaled_diff
          = (half * inv_L_dbl.template triangularView<Eigen::Lower>())
                .transpose();

      logp -= 0.5 * dot_self(half);

      if (!is_constant_all<T_y>::value) {
        ops_partials.edge1_.partials_vec_[i] -= scaled_diff;
      }
      if (!is_constant_all<T_loc>::value) {
        ops_partials.edge2_.partials_vec_[i] += scaled_diff;
      }
      if (!is_constant_all<T_covar>::value) {
        ops_partials.edge3_.partials_ += scaled_diff * half;
      }
    }
  }

  if (include_summand<propto, T_covar_elem>::value) {
    logp += sum(log(inv_L_dbl.diagonal())) * size_vec;
    if (!is_constant_all<T_covar>::value) {
      ops_partials.edge3_.partials_ -= size_vec * inv_L_dbl.transpose();
    }
  }

  return ops_partials.build(logp);
}

template <typename T_y, typename T_loc, typename T_covar>
inline return_type_t<T_y, T_loc, T_covar> multi_normal_cholesky_lpdf(
    const T_y& y, const T_loc& mu, const T_covar& L) {
  return multi_normal_cholesky_lpdf<false>(y, mu, L);
}

}  // namespace math
}  // namespace stan
#endif
<commit_msg>[Jenkins] auto-formatting by clang-format version 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)<commit_after>#ifndef STAN_MATH_PRIM_PROB_MULTI_NORMAL_CHOLESKY_LPDF_HPP
#define STAN_MATH_PRIM_PROB_MULTI_NORMAL_CHOLESKY_LPDF_HPP

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/err.hpp>
#include <stan/math/prim/fun/as_value_column_vector_or_scalar.hpp>
#include <stan/math/prim/fun/constants.hpp>
#include <stan/math/prim/fun/dot_self.hpp>
#include <stan/math/prim/fun/log.hpp>
#include <stan/math/prim/fun/max_size_mvt.hpp>
#include <stan/math/prim/fun/mdivide_left_tri.hpp>
#include <stan/math/prim/fun/size_mvt.hpp>
#include <stan/math/prim/fun/sum.hpp>
#include <stan/math/prim/fun/transpose.hpp>
#include <stan/math/prim/fun/vector_seq_view.hpp>
#include <stan/math/prim/functor/operands_and_partials.hpp>

namespace stan {
namespace math {

/** \ingroup multivar_dists
 * The log of the multivariate normal density for the given y, mu, and
 * a Cholesky factor L of the variance matrix.
 * Sigma = LL', a square, semi-positive definite matrix.
 *
 * Analytic expressions taken from
 * http://qwone.com/~jason/writing/multivariateNormal.pdf
 * written by Jason D. M. Rennie.
 *
 * @param y A scalar vector
 * @param mu The mean vector of the multivariate normal distribution.
 * @param L The Cholesky decomposition of a variance matrix
 * of the multivariate normal distribution
 * @return The log of the multivariate normal density.
 * @throw std::domain_error if LL' is not square, not symmetric,
 * or not semi-positive definite.
 * @tparam T_y Type of scalar.
 * @tparam T_loc Type of location.
 * @tparam T_covar Type of scale.
 */
template <bool propto, typename T_y, typename T_loc, typename T_covar,
          require_all_not_nonscalar_prim_or_rev_kernel_expression_t<
              T_y, T_loc, T_covar>* = nullptr>
return_type_t<T_y, T_loc, T_covar> multi_normal_cholesky_lpdf(
    const T_y& y, const T_loc& mu, const T_covar& L) {
  static const char* function = "multi_normal_cholesky_lpdf";
  using T_covar_elem = typename scalar_type<T_covar>::type;
  using T_return = return_type_t<T_y, T_loc, T_covar>;
  using T_partials_return = partials_return_t<T_y, T_loc, T_covar>;
  using matrix_partials_t
      = Eigen::Matrix<T_partials_return, Eigen::Dynamic, Eigen::Dynamic>;
  using vector_partials_t = Eigen::Matrix<T_partials_return, Eigen::Dynamic, 1>;
  using row_vector_partials_t
      = Eigen::Matrix<T_partials_return, 1, Eigen::Dynamic>;
  using T_y_ref = ref_type_t<T_y>;
  using T_mu_ref = ref_type_t<T_loc>;
  using T_L_ref = ref_type_t<T_covar>;

  check_consistent_sizes_mvt(function, "y", y, "mu", mu);
  size_t number_of_y = size_mvt(y);
  size_t number_of_mu = size_mvt(mu);
  if (number_of_y == 0 || number_of_mu == 0) {
    return 0;
  }

  T_y_ref y_ref = y;
  T_mu_ref mu_ref = mu;
  T_L_ref L_ref = L;
  vector_seq_view<T_y_ref> y_vec(y_ref);
  vector_seq_view<T_mu_ref> mu_vec(mu_ref);
  const size_t size_vec = max_size_mvt(y, mu);

  const int size_y = y_vec[0].size();
  const int size_mu = mu_vec[0].size();
  if (likely(size_vec > 1)) {
    // check size consistency of all random variables y
    for (size_t i = 1, size_mvt_y = size_mvt(y); i < size_mvt_y; i++) {
      check_size_match(function,
                       "Size of one of the vectors of "
                       "the random variable",
                       y_vec[i].size(),
                       "Size of the first vector of the "
                       "random variable",
                       size_y);
    }
    // check size consistency of all means mu
    for (size_t i = 1, size_mvt_mu = size_mvt(mu); i < size_mvt_mu; i++) {
      check_size_match(function,
                       "Size of one of the vectors of "
                       "the location variable",
                       mu_vec[i].size(),
                       "Size of the first vector of the "
                       "location variable",
                       size_mu);
    }
  }

  check_size_match(function, "Size of random variable", size_y,
                   "size of location parameter", size_mu);
  check_size_match(function, "Size of random variable", size_y,
                   "rows of covariance parameter", L.rows());
  check_size_match(function, "Size of random variable", size_y,
                   "columns of covariance parameter", L.cols());

  for (size_t i = 0; i < size_vec; i++) {
    check_finite(function, "Location parameter", mu_vec[i]);
    check_not_nan(function, "Random variable", y_vec[i]);
  }

  if (unlikely(size_y == 0)) {
    return T_return(0);
  }

  operands_and_partials<T_y_ref, T_mu_ref, T_L_ref> ops_partials(y_ref, mu_ref,
                                                                 L_ref);

  T_partials_return logp(0);
  if (include_summand<propto>::value) {
    logp += NEG_LOG_SQRT_TWO_PI * size_y * size_vec;
  }

  const matrix_partials_t inv_L_dbl
      = mdivide_left_tri<Eigen::Lower>(value_of(L_ref));

  if (include_summand<propto, T_y, T_loc, T_covar_elem>::value) {
    for (size_t i = 0; i < size_vec; i++) {
      decltype(auto) y_val = as_value_column_vector_or_scalar(y_vec[i]);
      decltype(auto) mu_val = as_value_column_vector_or_scalar(mu_vec[i]);

      const row_vector_partials_t half
          = (inv_L_dbl.template triangularView<Eigen::Lower>()
             * (y_val - mu_val).template cast<T_partials_return>())
                .transpose();
      const vector_partials_t scaled_diff
          = (half * inv_L_dbl.template triangularView<Eigen::Lower>())
                .transpose();

      logp -= 0.5 * dot_self(half);

      if (!is_constant_all<T_y>::value) {
        ops_partials.edge1_.partials_vec_[i] -= scaled_diff;
      }
      if (!is_constant_all<T_loc>::value) {
        ops_partials.edge2_.partials_vec_[i] += scaled_diff;
      }
      if (!is_constant_all<T_covar>::value) {
        ops_partials.edge3_.partials_ += scaled_diff * half;
      }
    }
  }

  if (include_summand<propto, T_covar_elem>::value) {
    logp += sum(log(inv_L_dbl.diagonal())) * size_vec;
    if (!is_constant_all<T_covar>::value) {
      ops_partials.edge3_.partials_ -= size_vec * inv_L_dbl.transpose();
    }
  }

  return ops_partials.build(logp);
}

template <typename T_y, typename T_loc, typename T_covar>
inline return_type_t<T_y, T_loc, T_covar> multi_normal_cholesky_lpdf(
    const T_y& y, const T_loc& mu, const T_covar& L) {
  return multi_normal_cholesky_lpdf<false>(y, mu, L);
}

}  // namespace math
}  // namespace stan
#endif
<|endoftext|>


<commit_before>/* A C++ wrapper around libev, which Rubinius uses for all event handling.
 * This is the folcrum of the thread subsystem. */

#include "prelude.hpp"
#include "object.hpp"
#include "objects.hpp"
#include "vm.hpp"
#include "objectmemory.hpp"

#include "builtin/tuple.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/io.hpp"

#include "event.hpp"

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <ev.h>

#include <vector>

namespace rubinius {
  namespace event {
    static void dispatch(Event *obj) {
      if(obj->activated()) {
        if(obj->loop) {
          obj->loop->remove_event(obj);
        }

        if(obj->tracked()) {
          delete obj;
        } else {
          obj->stop();
        }
      }
    }

    template <typename S>
      void tramp(EV_P_ S* ev, int revents) {
        event::dispatch((Event*)ev->data);
      }

    Event::Event(STATE, ObjectCallback* chan) :
      state(state), channel(chan), id(0),
      buffer(state), loop(NULL) { }

    void IO::stop() {
      ev_io_stop(loop->base, &ev);
    }

    void IO::start() {
      ev_io_start(loop->base, &ev);
    }

    bool IO::for_fd_p(int in) {
      if(fd == in) return true;
      return false;
    }

    Write::Write(STATE, ObjectCallback* chan, int fd) : IO(state, chan) {
      ev_io_init(&ev, event::tramp<struct ev_io>, fd, EV_WRITE);
      ev.data = this;
    }

    bool Write::activated() {
      channel->call(Qnil);
      return true;
    }

    Read::Read(STATE, ObjectCallback* chan, int ifd) :
        IO(state, chan), count(0) {
      fd = ifd;
      ev_io_init(&ev, event::tramp<struct ev_io>, fd, EV_READ);
      ev.data = this;
    }

    void Read::into_buffer(rubinius::IOBuffer *buf, size_t bytes) {
      count = bytes;
      buffer.set(buf);
    }

    bool Read::activated() {
      OBJECT ret;

      if(buffer->nil_p()) {
        ret = Object::i2n(state, fd);
      } else {
        /* the + 1 is for the null on the end */
        size_t bytes_to_read = count + 1;

        if(buffer->left() < bytes_to_read) {
          bytes_to_read = buffer->left() - 1;
        }

        char* start = buffer->at_unused();
        while(1) {
          int i = read(fd, start, bytes_to_read);

          /* EOF seen */
          if(i == 0) {
            ret = Qnil;

          /* didn't work out... */
          } else if(i == -1) {
            /* we were interrupted, how rude. go again. */
            if(errno == EINTR) continue;

            /* not sure. Send a system error */
            ret = Tuple::from(state, 2, state->symbol("error"), Object::i2n(errno));
          } else {
            /* clamp */
            start[i] = 0;

            buffer->read_bytes(i);
            ret = Object::i2n(i);
          }

          break;
        }
      }

      channel->call(ret);

      return true;
    }

    Signal::Signal(STATE, ObjectCallback *chan, int sig) :
        Event(state, chan), signal(sig) {
      ev_signal_init(&ev, event::tramp<struct ev_signal>, sig);
      ev.data = this;
    }

    void Signal::start() {
      loop->remove_signal(signal);
      loop->add_signal(signal, this);
      ev_signal_start(loop->base, &ev);
    }

    void Signal::stop() {
      ev_signal_stop(loop->base, &ev);
    }

    bool Signal::activated() {
      channel->call(state->current_thread());
      return false;
    }

    Timer::Timer(STATE, ObjectCallback* chan, double seconds) :
        Event(state, chan) {
      ev_timer_init(&ev, event::tramp<struct ev_timer>, (ev_tstamp)seconds, 0);
      ev.data = this;
    }

    void Timer::start() {
      ev_timer_start(loop->base, &ev);
    }

    void Timer::stop() {
      ev_timer_stop(loop->base, &ev);
    }

    bool Timer::activated() {
      channel->call(Qnil);
      return true;
    }

    Child::Child(STATE, ObjectCallback* chan, pid_t pid, int opts) :
      Event(state, chan), pid(pid), options(opts) { }

    bool Child::poll() {
      pid_t p;
      int status;
      OBJECT ret;

      while((p = waitpid(pid, &status, WNOHANG || options)) <= -1
          && errno == EINTR);

      if(p > 0) {
        if(WIFEXITED(status)) {
          ret = Object::i2n(WEXITSTATUS(status));
        } else {
          /* Could support WIFSIGNALED also. */
          ret = Qtrue;
        }
        channel->call(Tuple::from(state, 2, Object::i2n(p), ret));
      } else if(pid == -1 && errno == ECHILD) {
        channel->call(Qfalse);
      } else if(pid == 0 && (options & WNOHANG)) {
        channel->call(Qnil);
      } else {
        return false;
      }

      return true;
    }

    Loop::Loop(struct ev_loop *loop) :
      base(loop), event_ids(0), owner(false) { }

    Loop::Loop() : event_ids(0), owner(true) {
      base = ev_default_loop(0);
    }

    Loop::Loop(int opts) : event_ids(0), owner(true) {
      base = ev_loop_new(opts);
    }

    /* Gives this loop ownership of +ev+, letting it delete +ev+
     * when +ev+ is done, then starts the event. */
    void Loop::start(Event* ev) {
      ev->loop = this;
      ev->id = ++event_ids;
      events.push_back(ev);
      ev->start();
    }

    Loop::~Loop() {
      if(owner) ev_loop_destroy(base);
    }

    size_t Loop::num_of_events() {
      return events.size();
    }

    size_t Loop::loop_count() {
      return (size_t)ev_loop_count(base);
    }

    void Loop::poll() {
      ev_loop(base, EVLOOP_NONBLOCK);
    }

    void Loop::run_and_wait() {
      ev_loop(base, EVLOOP_ONESHOT);
    }

    void Loop::clear_by_fd(int fd) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->for_fd_p(fd)) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::clear_by_channel(void* chan) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->channel == chan) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::clear_by_id(size_t id) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->id == id) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::remove_event(Event *ev) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if(*it == ev) {
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::remove_signal(int sig) {
    }

    void Loop::add_signal(int sig, Signal* ev) {
    }
  } // event
} // rubinius
<commit_msg>Added thread dependency to vm/event.cpp<commit_after>/* A C++ wrapper around libev, which Rubinius uses for all event handling.
 * This is the folcrum of the thread subsystem. */

#include "prelude.hpp"
#include "object.hpp"
#include "objects.hpp"
#include "vm.hpp"
#include "objectmemory.hpp"

#include "builtin/tuple.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/io.hpp"
#include "builtin/thread.hpp"

#include "event.hpp"

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <ev.h>

#include <vector>

namespace rubinius {
  namespace event {
    static void dispatch(Event *obj) {
      if(obj->activated()) {
        if(obj->loop) {
          obj->loop->remove_event(obj);
        }

        if(obj->tracked()) {
          delete obj;
        } else {
          obj->stop();
        }
      }
    }

    template <typename S>
      void tramp(EV_P_ S* ev, int revents) {
        event::dispatch((Event*)ev->data);
      }

    Event::Event(STATE, ObjectCallback* chan) :
      state(state), channel(chan), id(0),
      buffer(state), loop(NULL) { }

    void IO::stop() {
      ev_io_stop(loop->base, &ev);
    }

    void IO::start() {
      ev_io_start(loop->base, &ev);
    }

    bool IO::for_fd_p(int in) {
      if(fd == in) return true;
      return false;
    }

    Write::Write(STATE, ObjectCallback* chan, int fd) : IO(state, chan) {
      ev_io_init(&ev, event::tramp<struct ev_io>, fd, EV_WRITE);
      ev.data = this;
    }

    bool Write::activated() {
      channel->call(Qnil);
      return true;
    }

    Read::Read(STATE, ObjectCallback* chan, int ifd) :
        IO(state, chan), count(0) {
      fd = ifd;
      ev_io_init(&ev, event::tramp<struct ev_io>, fd, EV_READ);
      ev.data = this;
    }

    void Read::into_buffer(rubinius::IOBuffer *buf, size_t bytes) {
      count = bytes;
      buffer.set(buf);
    }

    bool Read::activated() {
      OBJECT ret;

      if(buffer->nil_p()) {
        ret = Object::i2n(state, fd);
      } else {
        /* the + 1 is for the null on the end */
        size_t bytes_to_read = count + 1;

        if(buffer->left() < bytes_to_read) {
          bytes_to_read = buffer->left() - 1;
        }

        char* start = buffer->at_unused();
        while(1) {
          int i = read(fd, start, bytes_to_read);

          /* EOF seen */
          if(i == 0) {
            ret = Qnil;

          /* didn't work out... */
          } else if(i == -1) {
            /* we were interrupted, how rude. go again. */
            if(errno == EINTR) continue;

            /* not sure. Send a system error */
            ret = Tuple::from(state, 2, state->symbol("error"), Object::i2n(errno));
          } else {
            /* clamp */
            start[i] = 0;

            buffer->read_bytes(i);
            ret = Object::i2n(i);
          }

          break;
        }
      }

      channel->call(ret);

      return true;
    }

    Signal::Signal(STATE, ObjectCallback *chan, int sig) :
        Event(state, chan), signal(sig) {
      ev_signal_init(&ev, event::tramp<struct ev_signal>, sig);
      ev.data = this;
    }

    void Signal::start() {
      loop->remove_signal(signal);
      loop->add_signal(signal, this);
      ev_signal_start(loop->base, &ev);
    }

    void Signal::stop() {
      ev_signal_stop(loop->base, &ev);
    }

    bool Signal::activated() {
      channel->call(state->current_thread());
      return false;
    }

    Timer::Timer(STATE, ObjectCallback* chan, double seconds) :
        Event(state, chan) {
      ev_timer_init(&ev, event::tramp<struct ev_timer>, (ev_tstamp)seconds, 0);
      ev.data = this;
    }

    void Timer::start() {
      ev_timer_start(loop->base, &ev);
    }

    void Timer::stop() {
      ev_timer_stop(loop->base, &ev);
    }

    bool Timer::activated() {
      channel->call(Qnil);
      return true;
    }

    Child::Child(STATE, ObjectCallback* chan, pid_t pid, int opts) :
      Event(state, chan), pid(pid), options(opts) { }

    bool Child::poll() {
      pid_t p;
      int status;
      OBJECT ret;

      while((p = waitpid(pid, &status, WNOHANG || options)) <= -1
          && errno == EINTR);

      if(p > 0) {
        if(WIFEXITED(status)) {
          ret = Object::i2n(WEXITSTATUS(status));
        } else {
          /* Could support WIFSIGNALED also. */
          ret = Qtrue;
        }
        channel->call(Tuple::from(state, 2, Object::i2n(p), ret));
      } else if(pid == -1 && errno == ECHILD) {
        channel->call(Qfalse);
      } else if(pid == 0 && (options & WNOHANG)) {
        channel->call(Qnil);
      } else {
        return false;
      }

      return true;
    }

    Loop::Loop(struct ev_loop *loop) :
      base(loop), event_ids(0), owner(false) { }

    Loop::Loop() : event_ids(0), owner(true) {
      base = ev_default_loop(0);
    }

    Loop::Loop(int opts) : event_ids(0), owner(true) {
      base = ev_loop_new(opts);
    }

    /* Gives this loop ownership of +ev+, letting it delete +ev+
     * when +ev+ is done, then starts the event. */
    void Loop::start(Event* ev) {
      ev->loop = this;
      ev->id = ++event_ids;
      events.push_back(ev);
      ev->start();
    }

    Loop::~Loop() {
      if(owner) ev_loop_destroy(base);
    }

    size_t Loop::num_of_events() {
      return events.size();
    }

    size_t Loop::loop_count() {
      return (size_t)ev_loop_count(base);
    }

    void Loop::poll() {
      ev_loop(base, EVLOOP_NONBLOCK);
    }

    void Loop::run_and_wait() {
      ev_loop(base, EVLOOP_ONESHOT);
    }

    void Loop::clear_by_fd(int fd) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->for_fd_p(fd)) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::clear_by_channel(void* chan) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->channel == chan) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::clear_by_id(size_t id) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if((*it)->id == id) {
          delete *it;
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::remove_event(Event *ev) {
      std::vector<Event*>::iterator it;
      for(it = events.begin(); it != events.end();) {
        if(*it == ev) {
          it = events.erase(it);
        } else {
          it++;
        }
      }
    }

    void Loop::remove_signal(int sig) {
    }

    void Loop::add_signal(int sig, Signal* ev) {
    }
  } // event
} // rubinius
<|endoftext|>


<commit_before>/**
 * @file pca_main.cpp
 * @author Ryan Curtin
 *
 * Main executable to run PCA.
 */
#include <mlpack/core.hpp>

#include "pca.hpp"

using namespace mlpack;
using namespace mlpack::pca;
using namespace std;

// Document program.
PROGRAM_INFO("Principal Components Analysis", "This program performs principal "
    "components analysis on the given dataset.  It will transform the data "
    "onto its principal components, optionally performing dimensionality "
    "reduction by ignoring the principal components with the smallest "
    "eigenvalues.");

// Parameters for program.
PARAM_STRING_REQ("input_file", "Input dataset to perform PCA on.", "i");
PARAM_STRING_REQ("output_file", "File to save modified dataset to.", "o");
PARAM_INT("new_dimensionality", "Desired dimensionality of output dataset.  If "
    "0, no dimensionality reduction is performed.", "d", 0);

PARAM_FLAG("scale", "If set, the data will be scaled before running PCA, such "
    "that the variance of each feature is 1.", "s");

int main(int argc, char** argv)
{
  // Parse commandline.
  CLI::ParseCommandLine(argc, argv);

  // Load input dataset.
  string inputFile = CLI::GetParam<string>("input_file");
  arma::mat dataset;
  data::Load(inputFile.c_str(), dataset);

  // Find out what dimension we want.
  const size_t newDimension = dataset.n_rows; // No reduction, by default.
  if (CLI::GetParam<int>("new_dimensionality") != 0)
  {
    // Validate the parameter.
    newDimension = (size_t) CLI::GetParam<int>("new_dimensionality");
    if (newDimension > dataset.n_rows)
    {
      Log::Fatal << "New dimensionality (" << newDimension
          << ") cannot be greater than existing dimensionality ("
          << dataset.n_rows << ")!" << endl;
    }
  }

  // Get the options for running PCA.
  const size_t scale = CLI::HasParam("scale");

  // Perform PCA.
  PCA p(scale);
  Log::Info << "Performing PCA on dataset..." << endl;
  p.Apply(dataset, newDimension);

  // Now save the results.
  string outputFile = CLI::GetParam<string>("output_file");
  data::Save(outputFile, dataset);
}
<commit_msg>Stupid const error.  Did I not test it?<commit_after>/**
 * @file pca_main.cpp
 * @author Ryan Curtin
 *
 * Main executable to run PCA.
 */
#include <mlpack/core.hpp>

#include "pca.hpp"

using namespace mlpack;
using namespace mlpack::pca;
using namespace std;

// Document program.
PROGRAM_INFO("Principal Components Analysis", "This program performs principal "
    "components analysis on the given dataset.  It will transform the data "
    "onto its principal components, optionally performing dimensionality "
    "reduction by ignoring the principal components with the smallest "
    "eigenvalues.");

// Parameters for program.
PARAM_STRING_REQ("input_file", "Input dataset to perform PCA on.", "i");
PARAM_STRING_REQ("output_file", "File to save modified dataset to.", "o");
PARAM_INT("new_dimensionality", "Desired dimensionality of output dataset.  If "
    "0, no dimensionality reduction is performed.", "d", 0);

PARAM_FLAG("scale", "If set, the data will be scaled before running PCA, such "
    "that the variance of each feature is 1.", "s");

int main(int argc, char** argv)
{
  // Parse commandline.
  CLI::ParseCommandLine(argc, argv);

  // Load input dataset.
  string inputFile = CLI::GetParam<string>("input_file");
  arma::mat dataset;
  data::Load(inputFile.c_str(), dataset);

  // Find out what dimension we want.
  size_t newDimension = dataset.n_rows; // No reduction, by default.
  if (CLI::GetParam<int>("new_dimensionality") != 0)
  {
    // Validate the parameter.
    newDimension = (size_t) CLI::GetParam<int>("new_dimensionality");
    if (newDimension > dataset.n_rows)
    {
      Log::Fatal << "New dimensionality (" << newDimension
          << ") cannot be greater than existing dimensionality ("
          << dataset.n_rows << ")!" << endl;
    }
  }

  // Get the options for running PCA.
  const size_t scale = CLI::HasParam("scale");

  // Perform PCA.
  PCA p(scale);
  Log::Info << "Performing PCA on dataset..." << endl;
  p.Apply(dataset, newDimension);

  // Now save the results.
  string outputFile = CLI::GetParam<string>("output_file");
  data::Save(outputFile, dataset);
}
<|endoftext|>