<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/segment.hpp"

#include <caf/binary_deserializer.hpp>

#include "vast/bitmap.hpp"
#include "vast/bitmap_algorithms.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/byte_swap.hpp"
#include "vast/detail/narrow.hpp"
#include "vast/ids.hpp"
#include "vast/logger.hpp"
#include "vast/si_literals.hpp"
#include "vast/table_slice.hpp"

namespace vast {

using namespace binary_byte_literals;

segment_ptr segment::make(chunk_ptr chunk) {
  VAST_ASSERT(chunk != nullptr);
  // Setup a CAF deserializer
  caf::binary_deserializer source{nullptr, chunk->data(), chunk->size()};
  auto result = segment_ptr{new segment, false};
  if (auto error = source(result->header_, result->meta_)) {
    VAST_ERROR_ANON(__func__, "failed to deserialize segment meta data");
    return nullptr;
  }
  if (result->magic != magic) {
    VAST_ERROR_ANON(__func__, "got invalid segment magic", result->magic);
    return nullptr;
  }
  if (result->version > version) {
    VAST_ERROR_ANON(__func__, "got newer segment version", result->version);
    return nullptr;
  }
  // Skip meta data. Since the buffer following the chunk meta data was
  // previously serialized as chunk pointer (uint32_t size + data), we have
  // to add add sizeof(uint32_t) bytes to directly jump to the table slice
  // data.
  using detail::narrow_cast;
  auto bytes_read = chunk->size() - source.remaining();
  VAST_ASSERT(bytes_read < chunk->size());
  result->chunk_ = chunk->slice(bytes_read + sizeof(uint32_t));
  return result;
}

const uuid& segment::id() const {
  return header_.id;
}

chunk_ptr segment::chunk() const {
  return chunk_;
}

size_t segment::num_slices() const {
  return meta_.slices.size();
}

caf::expected<std::vector<table_slice_ptr>>
segment::lookup(const ids& xs) const {
  std::vector<table_slice_ptr> result;
  auto f = [](auto& slice) {
    return std::pair{slice.offset, slice.offset + slice.size};
  };
  auto g = [&](auto& slice) -> caf::error {
    auto x = make_slice(slice);
    if (!x)
      return x.error();
    result.push_back(*x);
    return caf::none;
  };
  auto begin = meta_.slices.begin();
  auto end = meta_.slices.end();
  if (auto error = select_with(xs, begin, end, f, g))
    return error;
  return result;
}

caf::expected<table_slice_ptr>
segment::make_slice(const table_slice_synopsis& slice) const {
  auto slice_size = detail::narrow_cast<size_t>(slice.end - slice.start);
  caf::binary_deserializer source{nullptr, chunk_->data() + slice.start,
                                  slice_size};
  table_slice_ptr result;
  if (auto error = source(result))
    return error;
  return result;
}

caf::error inspect(caf::serializer& sink, const segment_ptr& x) {
  VAST_ASSERT(x != nullptr);
  return sink(x->header_, x->meta_, x->chunk_);
}

caf::error inspect(caf::deserializer& source, segment_ptr& x) {
  x.reset(new segment, false);
  return source(x->header_, x->meta_, x->chunk_);
}

ids flat_slice_ids(const segment::meta_data& x) {
  ids result;
  for (auto& synopsis : x.slices) {
    result.append_bits(false, synopsis.offset - result.size());
    result.append_bits(true, synopsis.size);
  }
  return result;
}

} // namespace vast
<commit_msg>Compare segment header version and magic<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/segment.hpp"

#include <caf/binary_deserializer.hpp>

#include "vast/bitmap.hpp"
#include "vast/bitmap_algorithms.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/byte_swap.hpp"
#include "vast/detail/narrow.hpp"
#include "vast/ids.hpp"
#include "vast/logger.hpp"
#include "vast/si_literals.hpp"
#include "vast/table_slice.hpp"

namespace vast {

using namespace binary_byte_literals;

segment_ptr segment::make(chunk_ptr chunk) {
  VAST_ASSERT(chunk != nullptr);
  // Setup a CAF deserializer
  caf::binary_deserializer source{nullptr, chunk->data(), chunk->size()};
  auto result = segment_ptr{new segment, false};
  if (auto error = source(result->header_, result->meta_)) {
    VAST_ERROR_ANON(__func__, "failed to deserialize segment meta data");
    return nullptr;
  }
  if (result->header_.magic != magic) {
    VAST_ERROR_ANON(__func__, "got invalid segment magic",
                    result->header_.magic, "instead of", magic);
    return nullptr;
  }
  if (result->header_.version > version) {
    VAST_ERROR_ANON(__func__, "got newer segment version",
                    result->header_.version, "instead of", version);
    return nullptr;
  }
  // Skip meta data. Since the buffer following the chunk meta data was
  // previously serialized as chunk pointer (uint32_t size + data), we have
  // to add add sizeof(uint32_t) bytes to directly jump to the table slice
  // data.
  using detail::narrow_cast;
  auto bytes_read = chunk->size() - source.remaining();
  VAST_ASSERT(bytes_read < chunk->size());
  result->chunk_ = chunk->slice(bytes_read + sizeof(uint32_t));
  return result;
}

const uuid& segment::id() const {
  return header_.id;
}

chunk_ptr segment::chunk() const {
  return chunk_;
}

size_t segment::num_slices() const {
  return meta_.slices.size();
}

caf::expected<std::vector<table_slice_ptr>>
segment::lookup(const ids& xs) const {
  std::vector<table_slice_ptr> result;
  auto f = [](auto& slice) {
    return std::pair{slice.offset, slice.offset + slice.size};
  };
  auto g = [&](auto& slice) -> caf::error {
    auto x = make_slice(slice);
    if (!x)
      return x.error();
    result.push_back(*x);
    return caf::none;
  };
  auto begin = meta_.slices.begin();
  auto end = meta_.slices.end();
  if (auto error = select_with(xs, begin, end, f, g))
    return error;
  return result;
}

caf::expected<table_slice_ptr>
segment::make_slice(const table_slice_synopsis& slice) const {
  auto slice_size = detail::narrow_cast<size_t>(slice.end - slice.start);
  caf::binary_deserializer source{nullptr, chunk_->data() + slice.start,
                                  slice_size};
  table_slice_ptr result;
  if (auto error = source(result))
    return error;
  return result;
}

caf::error inspect(caf::serializer& sink, const segment_ptr& x) {
  VAST_ASSERT(x != nullptr);
  return sink(x->header_, x->meta_, x->chunk_);
}

caf::error inspect(caf::deserializer& source, segment_ptr& x) {
  x.reset(new segment, false);
  return source(x->header_, x->meta_, x->chunk_);
}

ids flat_slice_ids(const segment::meta_data& x) {
  ids result;
  for (auto& synopsis : x.slices) {
    result.append_bits(false, synopsis.offset - result.size());
    result.append_bits(true, synopsis.size);
  }
  return result;
}

} // namespace vast
<|endoftext|>


<commit_before>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#ifndef GUARD_TENSOR_HOLDER_HPP
#define GUARD_TENSOR_HOLDER_HPP

#include "ford.hpp"
#include "network_data.hpp"
#include <miopen/tensor.hpp>
#include <miopen/functional.hpp>

template <class F>
void visit_tensor_size(std::size_t n, F f)
{
    switch(n)
    {
    case 0:
    {
        f(std::integral_constant<std::size_t, 0>{});
        break;
    }
    case 1:
    {
        f(std::integral_constant<std::size_t, 1>{});
        break;
    }
    case 2:
    {
        f(std::integral_constant<std::size_t, 2>{});
        break;
    }
    case 3:
    {
        f(std::integral_constant<std::size_t, 3>{});
        break;
    }
    case 4:
    {
        f(std::integral_constant<std::size_t, 4>{});
        break;
    }
    case 5:
    {
        f(std::integral_constant<std::size_t, 5>{});
        break;
    }
    default: throw std::runtime_error("Unknown tensor size");
    }
}

template <class T>
struct tensor
{
    miopen::TensorDescriptor desc;
    std::vector<T> data;

    tensor() {}

    template <class X>
    tensor(const std::vector<X>& dims)
        : desc(miopenFloat, dims.data(), static_cast<int>(dims.size())), data(desc.GetElementSize())
    {
    }

    tensor(std::size_t n, std::size_t c, std::size_t h, std::size_t w)
        : desc(miopenFloat, {n, c, h, w}), data(n * c * h * w)
    {
    }

    tensor(miopen::TensorDescriptor rhs) : desc(std::move(rhs))
    {
        data.resize(desc.GetElementSize());
    }

    template <class G>
    tensor& generate(G g) &
    {
        this->generate_impl(g);
        return *this;
    }

    template <class G>
    tensor&& generate(G g) &&
    {
        this->generate_impl(g);
        return std::move(*this);
    }

    template <class G>
    void generate_impl(G g)
    {
        auto iterator = data.begin();
        auto assign = [&](T x)
        {
           assert(iterator < data.end());
            *iterator = x;
            ++iterator;
        };
        this->for_each(miopen::compose(assign, std::move(g)));
    }

    template <class Loop, class F>
    struct for_each_unpacked
    {
        Loop loop;
        F f;
        template <class... Ts>
        auto operator()(Ts... xs) const -> decltype(f(xs...), void())
        {
            loop(xs...)(std::move(f));
        }

        void operator()(...) const
        {
            throw std::runtime_error("Arguments to for_each do not match tensor size");
        }
    };

    struct for_each_handler
    {
        template <class Self, class Loop, class F, class Size>
        void operator()(Self* self, Loop loop, F f, Size size) const
        {
            auto dims = miopen::tien<size>(self->desc.GetLengths());
            miopen::unpack(for_each_unpacked<Loop, F>{loop, std::move(f)}, dims);
        }
    };

    template <class F>
    void for_each(F f) const
    {
        visit_tensor_size(
            desc.GetLengths().size(),
            std::bind(for_each_handler{}, this, ford, std::move(f), std::placeholders::_1));
    }

    template <class F>
    void par_for_each(F f) const
    {
        visit_tensor_size(
            desc.GetLengths().size(),
            std::bind(for_each_handler{}, this, par_ford, std::move(f), std::placeholders::_1));
    }

    template <class... Ts>
    T& operator()(Ts... ds)
    {
        assert(this->desc.GetIndex(ds...) < data.size());
        return this->data[this->desc.GetIndex(ds...)];
    }

    template <class... Ts>
    const T& operator()(Ts... xs) const
    {
        assert(this->desc.GetIndex(xs...) < data.size());
        return this->data[this->desc.GetIndex(xs...)];
    }

    T& operator[](std::size_t i) { return data.at(i); }

    const T& operator[](std::size_t i) const { return data.at(i); }

    typename std::vector<T>::iterator begin() { return data.begin(); }

    typename std::vector<T>::iterator end() { return data.end(); }

    typename std::vector<T>::const_iterator begin() const { return data.begin(); }

    typename std::vector<T>::const_iterator end() const { return data.end(); }
};

template <class T, class G>
tensor<T> make_tensor(std::initializer_list<std::size_t> dims, G g)
{
    // TODO: Compute float
    return tensor<T>{miopen::TensorDescriptor{miopenFloat, dims}}.generate(g);
}

template <class T, class X>
tensor<T> make_tensor(const std::vector<X>& dims)
{
    // TODO: Compute float
    return tensor<T>{
        miopen::TensorDescriptor{miopenFloat, dims.data(), static_cast<int>(dims.size())}};
}

template <class T, class X, class G>
tensor<T> make_tensor(const std::vector<X>& dims, G g)
{
    return make_tensor<T>(dims).generate(g);
}

struct tensor_generate
{
    template <class Tensor, class G>
    Tensor&& operator()(Tensor&& t, G g) const
    {
        return std::forward<Tensor>(t.generate(g));
    }
};

template <class F>
struct protect_void_fn
{
    F f;
    protect_void_fn(F x) : f(std::move(x)) {}

    // template<class... Ts>
    // auto operator()(Ts&&... xs) const MIOPEN_RETURNS
    // (f(std::forward<Ts>(xs)...));

    template <class... Ts>
    void operator()(Ts&&... xs) const
    {
        f(std::forward<Ts>(xs)...);
    }
};

template <class F>
protect_void_fn<F> protect_void(F f)
{
    return {std::move(f)};
}

struct cross_args_apply
{
    template <class F, class T, class... Ts>
    void operator()(F f, T&& x, Ts&&... xs) const
    {
        miopen::each_args(std::bind(f, std::forward<T>(x), std::placeholders::_1),
                          std::forward<Ts>(xs)...);
    }
};

template <class F, class... Ts>
void cross_args(F f, Ts&&... xs)
{
    miopen::each_args(std::bind(cross_args_apply{},
                                protect_void(std::move(f)),
                                std::placeholders::_1,
                                std::forward<Ts>(xs)...),
                      std::forward<Ts>(xs)...);
}

template <class T>
struct generate_both_visitation
{
    template <class F, class G1, class G2>
    void operator()(F f, G1 g1, G2 g2) const
    {
        for(auto&& input : get_inputs())
            for(auto&& weights : get_weights())
                if(input.at(1) == weights.at(1)) // channels must match
                    f(make_tensor<T>(input, g1), make_tensor<T>(weights, g2));
    }
};

template <class T, class F, class... Gs>
void generate_binary_all(F f, Gs... gs)
{
    cross_args(std::bind(generate_both_visitation<T>{},
                         protect_void(f),
                         std::placeholders::_1,
                         std::placeholders::_2),
               gs...);
}

template <class T, class F, class G>
void generate_binary_one(F f, std::vector<int> input, std::vector<int> weights, G g)
{
    f(make_tensor<T>(input, g), make_tensor<T>(weights, g));
}

template <class T>
struct generate_activ_visitation
{
    template <class F, class G>
    void operator()(F f, G g) const
    {
        for(auto&& input : get_inputs())
            f(make_tensor<T>(input, g));
    }
};

template <class T, class F, class... Gs>
void generate_unary_all(F f, Gs... gs)
{
    miopen::each_args(
        std::bind(generate_activ_visitation<T>{}, protect_void(f), std::placeholders::_1), gs...);
}

template <class T, class F, class G>
void generate_unary_one(F f, std::vector<int> input, G g)
{
    f(make_tensor<T>(input, g));
}

#endif
<commit_msg>Finally worked.<commit_after>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#ifndef GUARD_TENSOR_HOLDER_HPP
#define GUARD_TENSOR_HOLDER_HPP

#include "ford.hpp"
#include "network_data.hpp"
#include <miopen/tensor.hpp>
#include <miopen/functional.hpp>

template <class F>
void visit_tensor_size(std::size_t n, F f)
{
    switch(n)
    {
    case 0:
    {
        f(std::integral_constant<std::size_t, 0>{});
        break;
    }
    case 1:
    {
        f(std::integral_constant<std::size_t, 1>{});
        break;
    }
    case 2:
    {
        f(std::integral_constant<std::size_t, 2>{});
        break;
    }
    case 3:
    {
        f(std::integral_constant<std::size_t, 3>{});
        break;
    }
    case 4:
    {
        f(std::integral_constant<std::size_t, 4>{});
        break;
    }
    case 5:
    {
        f(std::integral_constant<std::size_t, 5>{});
        break;
    }
    default: throw std::runtime_error("Unknown tensor size");
    }
}

template <class T>
struct tensor
{
    miopen::TensorDescriptor desc;
    std::vector<T> data;

    tensor() {}

    template <class X>
    tensor(const std::vector<X>& dims)
        : desc(miopenFloat, dims.data(), static_cast<int>(dims.size())), data(desc.GetElementSize())
    {
    }

    tensor(std::size_t n, std::size_t c, std::size_t h, std::size_t w)
        : desc(miopenFloat, {n, c, h, w}), data(n * c * h * w)
    {
    }

    tensor(miopen::TensorDescriptor rhs) : desc(std::move(rhs))
    {
        data.resize(desc.GetElementSize());
    }

    template <class G>
    tensor& generate(G g) &
    {
        this->generate_impl(g);
        return *this;
    }

    template <class G>
    tensor&& generate(G g) &&
    {
        this->generate_impl(g);
        return std::move(*this);
    }

    template <class G>
    void generate_impl(G g)
    {
        auto iterator = data.begin();
        auto assign = [&](T x)
        {
           assert(iterator < data.end());
            *iterator = x;
            ++iterator;
        };
        this->for_each(miopen::compose(assign, std::move(g)));
    }

    template <class Loop, class F>
    struct for_each_unpacked
    {
        Loop loop;
        F f;
        template <class... Ts>
        auto operator()(Ts... xs) const -> decltype(f(xs...), void())
        {
            loop(xs...)(std::move(f));
        }

        void operator()(...) const
        {
            throw std::runtime_error("Arguments to for_each do not match tensor size");
        }
    };

    struct for_each_handler
    {
        template <class Self, class Loop, class F, class Size>
        void operator()(Self* self, Loop loop, F f, Size size) const
        {
            auto dims = miopen::tien<size>(self->desc.GetLengths());
            miopen::unpack(for_each_unpacked<Loop, F>{loop, std::move(f)}, dims);
        }
    };

    template <class F>
    void for_each(F f) const
    {
        visit_tensor_size(
            desc.GetLengths().size(),
            std::bind(for_each_handler{}, this, ford, std::move(f), std::placeholders::_1));
    }

    template <class F>
    void par_for_each(F f) const
    {
        visit_tensor_size(
            desc.GetLengths().size(),
            std::bind(for_each_handler{}, this, par_ford, std::move(f), std::placeholders::_1));
    }

    template <class... Ts>
    T& operator()(Ts... ds)
    {
        assert(this->desc.GetIndex(ds...) < data.size());
        return this->data[this->desc.GetIndex(ds...)];
    }

    template <class... Ts>
    const T& operator()(Ts... ds) const
    {
        assert(this->desc.GetIndex(ds...) < data.size());
        return this->data[this->desc.GetIndex(ds...)];
    }

    T& operator[](std::size_t i) { return data.at(i); }

    const T& operator[](std::size_t i) const { return data.at(i); }

    typename std::vector<T>::iterator begin() { return data.begin(); }

    typename std::vector<T>::iterator end() { return data.end(); }

    typename std::vector<T>::const_iterator begin() const { return data.begin(); }

    typename std::vector<T>::const_iterator end() const { return data.end(); }
};

template <class T, class G>
tensor<T> make_tensor(std::initializer_list<std::size_t> dims, G g)
{
    // TODO: Compute float
    return tensor<T>{miopen::TensorDescriptor{miopenFloat, dims}}.generate(g);
}

template <class T, class X>
tensor<T> make_tensor(const std::vector<X>& dims)
{
    // TODO: Compute float
    return tensor<T>{
        miopen::TensorDescriptor{miopenFloat, dims.data(), static_cast<int>(dims.size())}};
}

template <class T, class X, class G>
tensor<T> make_tensor(const std::vector<X>& dims, G g)
{
    return make_tensor<T>(dims).generate(g);
}

struct tensor_generate
{
    template <class Tensor, class G>
    Tensor&& operator()(Tensor&& t, G g) const
    {
        return std::forward<Tensor>(t.generate(g));
    }
};

template <class F>
struct protect_void_fn
{
    F f;
    protect_void_fn(F x) : f(std::move(x)) {}

    // template<class... Ts>
    // auto operator()(Ts&&... xs) const MIOPEN_RETURNS
    // (f(std::forward<Ts>(xs)...));

    template <class... Ts>
    void operator()(Ts&&... xs) const
    {
        f(std::forward<Ts>(xs)...);
    }
};

template <class F>
protect_void_fn<F> protect_void(F f)
{
    return {std::move(f)};
}

struct cross_args_apply
{
    template <class F, class T, class... Ts>
    void operator()(F f, T&& x, Ts&&... xs) const
    {
        miopen::each_args(std::bind(f, std::forward<T>(x), std::placeholders::_1),
                          std::forward<Ts>(xs)...);
    }
};

template <class F, class... Ts>
void cross_args(F f, Ts&&... xs)
{
    miopen::each_args(std::bind(cross_args_apply{},
                                protect_void(std::move(f)),
                                std::placeholders::_1,
                                std::forward<Ts>(xs)...),
                      std::forward<Ts>(xs)...);
}

template <class T>
struct generate_both_visitation
{
    template <class F, class G1, class G2>
    void operator()(F f, G1 g1, G2 g2) const
    {
        for(auto&& input : get_inputs())
            for(auto&& weights : get_weights())
                if(input.at(1) == weights.at(1)) // channels must match
                    f(make_tensor<T>(input, g1), make_tensor<T>(weights, g2));
    }
};

template <class T, class F, class... Gs>
void generate_binary_all(F f, Gs... gs)
{
    cross_args(std::bind(generate_both_visitation<T>{},
                         protect_void(f),
                         std::placeholders::_1,
                         std::placeholders::_2),
               gs...);
}

template <class T, class F, class G>
void generate_binary_one(F f, std::vector<int> input, std::vector<int> weights, G g)
{
    f(make_tensor<T>(input, g), make_tensor<T>(weights, g));
}

template <class T>
struct generate_activ_visitation
{
    template <class F, class G>
    void operator()(F f, G g) const
    {
        for(auto&& input : get_inputs())
            f(make_tensor<T>(input, g));
    }
};

template <class T, class F, class... Gs>
void generate_unary_all(F f, Gs... gs)
{
    miopen::each_args(
        std::bind(generate_activ_visitation<T>{}, protect_void(f), std::placeholders::_1), gs...);
}

template <class T, class F, class G>
void generate_unary_one(F f, std::vector<int> input, G g)
{
    f(make_tensor<T>(input, g));
}

#endif
<|endoftext|>


<commit_before>#include <iostream>
#include <vector>
#include <functional>
#include <UnitTest++.h>
#include <histogram.h>

TEST(ShouldUseAccessors)
{
    Histogram h{ 100000000, 3 };

    CHECK_EQUAL(3, h.getNumberOfSignificantValueDigits());
    CHECK_EQUAL(100000000, h.getHighestTrackableValue());
}

TEST(ShouldRecordValue)
{
    uint64_t testValue = 4;
    Histogram h{ 100000000, 3 };

    h.recordValue(testValue);
    CHECK_EQUAL(1, h.getCountAtValue(testValue));
    CHECK_EQUAL(1, h.getTotalCount());
}

const uint64_t HIGHEST_TRACKABLE_VALUE = 3600000000;
const uint64_t SIGNIFICANT_DIGITS = 3;

void loadHistograms(Histogram& histogramA, Histogram& histogramB)
{
    for (int i = 0; i < 10000; i++)
    {
        histogramA.recordValue(1000L);
        histogramB.recordValue(1000L, 10000L);
    }

    histogramA.recordValue(100000000L);
    histogramB.recordValue(100000000L, 10000L);
}

TEST(ShouldGetTotalCount)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_EQUAL(10001, histogram.getTotalCount());
    CHECK_EQUAL(20000, histogramCorrected.getTotalCount());
}

TEST(ShouldGetMaxValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK(histogram.valuesAreEquivalent(100000000L, histogram.getMaxValue()));
    CHECK(histogramCorrected.valuesAreEquivalent(100000000L, histogramCorrected.getMaxValue()));
}

TEST(ShouldGetMinValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK(histogram.valuesAreEquivalent(1000L, histogram.getMinValue()));
    CHECK(histogramCorrected.valuesAreEquivalent(1000L, histogramCorrected.getMinValue()));
}

TEST(ShouldGetMeanValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    double expectedMean = ((1000L * 10000.0) + (100000000L * 1.0))/10001;
    CHECK_CLOSE(expectedMean, histogram.getMeanValue(), expectedMean * 0.001);

    double expectedCorrectedMean = (1000.0 + 50000000.0)/2;
    CHECK_CLOSE(expectedCorrectedMean, histogramCorrected.getMeanValue(), expectedCorrectedMean * 0.001);
}

TEST(ShouldGetPercentileValues)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(30.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(99.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(99.99),
                1000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogram.getValueAtPercentile(99.999),
                100000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogram.getValueAtPercentile(100.0),
                100000000.0 * 0.001);

    CHECK_CLOSE(1000.0, (double) histogramCorrected.getValueAtPercentile(30.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogramCorrected.getValueAtPercentile(50.0),
                1000.0 * 0.001);
    CHECK_CLOSE(50000000.0, (double) histogramCorrected.getValueAtPercentile(75.0),
                50000000.0 * 0.001);
    CHECK_CLOSE(80000000.0, (double) histogramCorrected.getValueAtPercentile(90.0),
                80000000.0 * 0.001);
    CHECK_CLOSE(98000000.0, (double) histogramCorrected.getValueAtPercentile(99.0),
                98000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogramCorrected.getValueAtPercentile(99.999),
                100000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogramCorrected.getValueAtPercentile(100.0),
                100000000.0 * 0.001);
}

TEST(ShouldGetPercentileAtOrBelowValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_CLOSE(99.99, histogram.getPercentileAtOrBelowValue(5000),                0.0001);
    CHECK_CLOSE(50.0,  histogramCorrected.getPercentileAtOrBelowValue(5000),       0.0001);
    CHECK_CLOSE(100.0, histogramCorrected.getPercentileAtOrBelowValue(100000000L), 0.0001);
}

TEST(ShouldGetCountBetweenValues)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_EQUAL(10000, histogram.getCountBetweenValues(1000L, 1000L));
    CHECK_EQUAL(1,     histogram.getCountBetweenValues(5000L, 150000000L));
    CHECK_EQUAL(10000, histogramCorrected.getCountBetweenValues(5000L, 150000000L));
}
<commit_msg>Add test for getCountAtValue()<commit_after>#include <iostream>
#include <vector>
#include <functional>
#include <UnitTest++.h>
#include <histogram.h>

TEST(ShouldUseAccessors)
{
    Histogram h{ 100000000, 3 };

    CHECK_EQUAL(3, h.getNumberOfSignificantValueDigits());
    CHECK_EQUAL(100000000, h.getHighestTrackableValue());
}

TEST(ShouldRecordValue)
{
    uint64_t testValue = 4;
    Histogram h{ 100000000, 3 };

    h.recordValue(testValue);
    CHECK_EQUAL(1, h.getCountAtValue(testValue));
    CHECK_EQUAL(1, h.getTotalCount());
}

const uint64_t HIGHEST_TRACKABLE_VALUE = 3600000000;
const uint64_t SIGNIFICANT_DIGITS = 3;

void loadHistograms(Histogram& histogramA, Histogram& histogramB)
{
    for (int i = 0; i < 10000; i++)
    {
        histogramA.recordValue(1000L);
        histogramB.recordValue(1000L, 10000L);
    }

    histogramA.recordValue(100000000L);
    histogramB.recordValue(100000000L, 10000L);
}

TEST(ShouldGetTotalCount)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_EQUAL(10001, histogram.getTotalCount());
    CHECK_EQUAL(20000, histogramCorrected.getTotalCount());
}

TEST(ShouldGetMaxValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK(histogram.valuesAreEquivalent(100000000L, histogram.getMaxValue()));
    CHECK(histogramCorrected.valuesAreEquivalent(100000000L, histogramCorrected.getMaxValue()));
}

TEST(ShouldGetMinValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK(histogram.valuesAreEquivalent(1000L, histogram.getMinValue()));
    CHECK(histogramCorrected.valuesAreEquivalent(1000L, histogramCorrected.getMinValue()));
}

TEST(ShouldGetMeanValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    double expectedMean = ((1000L * 10000.0) + (100000000L * 1.0))/10001;
    CHECK_CLOSE(expectedMean, histogram.getMeanValue(), expectedMean * 0.001);

    double expectedCorrectedMean = (1000.0 + 50000000.0)/2;
    CHECK_CLOSE(expectedCorrectedMean, histogramCorrected.getMeanValue(), expectedCorrectedMean * 0.001);
}

TEST(ShouldGetPercentileValues)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(30.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(99.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogram.getValueAtPercentile(99.99),
                1000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogram.getValueAtPercentile(99.999),
                100000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogram.getValueAtPercentile(100.0),
                100000000.0 * 0.001);

    CHECK_CLOSE(1000.0, (double) histogramCorrected.getValueAtPercentile(30.0),
                1000.0 * 0.001);
    CHECK_CLOSE(1000.0, (double) histogramCorrected.getValueAtPercentile(50.0),
                1000.0 * 0.001);
    CHECK_CLOSE(50000000.0, (double) histogramCorrected.getValueAtPercentile(75.0),
                50000000.0 * 0.001);
    CHECK_CLOSE(80000000.0, (double) histogramCorrected.getValueAtPercentile(90.0),
                80000000.0 * 0.001);
    CHECK_CLOSE(98000000.0, (double) histogramCorrected.getValueAtPercentile(99.0),
                98000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogramCorrected.getValueAtPercentile(99.999),
                100000000.0 * 0.001);
    CHECK_CLOSE(100000000.0, (double) histogramCorrected.getValueAtPercentile(100.0),
                100000000.0 * 0.001);
}

TEST(ShouldGetPercentileAtOrBelowValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_CLOSE(99.99, histogram.getPercentileAtOrBelowValue(5000),                0.0001);
    CHECK_CLOSE(50.0,  histogramCorrected.getPercentileAtOrBelowValue(5000),       0.0001);
    CHECK_CLOSE(100.0, histogramCorrected.getPercentileAtOrBelowValue(100000000L), 0.0001);
}

TEST(ShouldGetCountBetweenValues)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_EQUAL(10000, histogram.getCountBetweenValues(1000L, 1000L));
    CHECK_EQUAL(1,     histogram.getCountBetweenValues(5000L, 150000000L));
    CHECK_EQUAL(10000, histogramCorrected.getCountBetweenValues(5000L, 150000000L));
}

TEST(ShouldGetCountAtValue)
{
    Histogram histogram{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    Histogram histogramCorrected{ HIGHEST_TRACKABLE_VALUE, SIGNIFICANT_DIGITS };
    loadHistograms(histogram, histogramCorrected);

    CHECK_EQUAL(0,     histogram.getCountBetweenValues(10000L, 10010L));
    CHECK_EQUAL(1,     histogramCorrected.getCountBetweenValues(10000L, 10010L));
    CHECK_EQUAL(10000, histogram.getCountAtValue(1000L));
    CHECK_EQUAL(10000, histogramCorrected.getCountAtValue(1000L));
}

<|endoftext|>


<commit_before>// Copyright (c) 2003 Daniel Wallin and Arvid Norberg

// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
// ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
// OR OTHER DEALINGS IN THE SOFTWARE.

#include "test.hpp"

#include <luabind/out_value_policy.hpp>
#include <luabind/return_reference_to_policy.hpp>
#include <luabind/copy_policy.hpp>
#include <luabind/adopt_policy.hpp>
#include <luabind/discard_result_policy.hpp>
#include <luabind/dependency_policy.hpp>
#include <luabind/luabind.hpp>

struct test_copy {};

struct secret_type {};

secret_type sec_;

struct policies_test_class
{
	policies_test_class(const char* name): name_(name)
	{ ++count; }
	policies_test_class() { ++count; }
	~policies_test_class()
	{ --count; }

	std::string name_;

	policies_test_class* make(const char* name) const
	{
		return new policies_test_class(name);
	}

	void f(policies_test_class* p)
	{
		delete p;
	}
	const policies_test_class* internal_ref() { return this; }
	policies_test_class* self_ref()
	{ return this; }

	static int count;

	//	private:
	policies_test_class(policies_test_class const& c): name_(c.name_)
	{ ++count; }

	void member_out_val(int a, int* v) { *v = a * 2; }
	secret_type* member_secret() { return &sec_; }
};

int policies_test_class::count = 0;

policies_test_class global;

void out_val(float* f) { *f = 3.f; }
policies_test_class* copy_val() { return &global; }
policies_test_class const* copy_val_const() { return &global; }

secret_type* secret() { return &sec_; }

void aux_test();

struct test_t {
	test_t *make(int) { return new test_t(); }
	void take(test_t*) {}
};

struct MI2;

struct MI1
{
	void add(MI2 *) {}
};

struct MI2 : public MI1 {};
struct MI2W : public MI2, public luabind::wrap_base {};

void test_main(lua_State* L)
{
	using namespace luabind;

	module(L)
	[
		class_<test_t>("test_t")
		.def("make", &test_t::make, adopt(return_value))
		.def("take", &test_t::take, adopt(_2))
	];

	module(L)
	[
		class_<policies_test_class>("test")
			.def(constructor<>())
			.def("member_out_val", &policies_test_class::member_out_val, pure_out_value(_3))
			.def("member_secret", &policies_test_class::member_secret, discard_result)
			.def("f", &policies_test_class::f, adopt(_2))
			.def("make", &policies_test_class::make, adopt(return_value))
			.def("internal_ref", &policies_test_class::internal_ref, dependency(result, _1))
			.def("self_ref", &policies_test_class::self_ref, return_reference_to(_1)),

		def("out_val", &out_val, pure_out_value(_1)),
		def("copy_val", &copy_val, copy(result)),
		def("copy_val_const", &copy_val_const, copy(result)),
		def("secret", &secret, discard_result),

		class_<MI1>("mi1")
			.def(constructor<>())
			.def("add",&MI1::add,adopt(_2)),

		class_<MI2,MI2W,MI1>("mi2")
			.def(constructor<>())
	];

	// test copy
	DOSTRING(L,	"a = secret()\n");

	TEST_CHECK(policies_test_class::count == 1);

	DOSTRING(L, "a = copy_val()\n");
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "b = copy_val_const()\n");
	TEST_CHECK(policies_test_class::count == 3);

	DOSTRING(L,
		"a = nil\n"
		"b = nil\n"
		"collectgarbage()\n");

	// only the global variable left here
	TEST_CHECK(policies_test_class::count == 1);

	// out_value
	DOSTRING(L,
		"a = out_val()\n"
		"assert(a == 3)");

	// return_reference_to
	DOSTRING(L,
		"a = test()\n"
		"b = a:self_ref()\n"
		"a = nil\n"
		"collectgarbage()");

	// a is kept alive as long as b is alive
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L,
		"b = nil\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	DOSTRING(L, "a = test()");

	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L,
		"b = a:internal_ref()\n"
		"a = nil\n"
		"collectgarbage()");

	// a is kept alive as long as b is alive
	TEST_CHECK(policies_test_class::count == 2);

	// two gc-cycles because dependency-table won't be collected in the
	// same cycle as the object_rep
	DOSTRING(L,
		"b = nil\n"
		"collectgarbage()\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	// adopt
	DOSTRING(L, "a = test()");

	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "b = a:make('tjosan')");
	DOSTRING(L, "assert(a:member_out_val(3) == 6)");
	DOSTRING(L, "a:member_secret()");

	// make instantiated a new policies_test_class
	TEST_CHECK(policies_test_class::count == 3);

	DOSTRING(L, "a:f(b)\n");

	// b was adopted by c++ and deleted the object
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "a = nil\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	// adopt with wrappers
	DOSTRING(L, "mi1():add(mi2())");
}

<commit_msg>Change test to use polymorphic type derived from wrap_base.<commit_after>// Copyright (c) 2003 Daniel Wallin and Arvid Norberg

// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
// ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
// OR OTHER DEALINGS IN THE SOFTWARE.

#include "test.hpp"

#include <luabind/out_value_policy.hpp>
#include <luabind/return_reference_to_policy.hpp>
#include <luabind/copy_policy.hpp>
#include <luabind/adopt_policy.hpp>
#include <luabind/discard_result_policy.hpp>
#include <luabind/dependency_policy.hpp>
#include <luabind/luabind.hpp>

struct test_copy {};

struct secret_type {};

secret_type sec_;

struct policies_test_class
{
	policies_test_class(const char* name): name_(name)
	{ ++count; }
	policies_test_class() { ++count; }
	~policies_test_class()
	{ --count; }

	std::string name_;

	policies_test_class* make(const char* name) const
	{
		return new policies_test_class(name);
	}

	void f(policies_test_class* p)
	{
		delete p;
	}
	const policies_test_class* internal_ref() { return this; }
	policies_test_class* self_ref()
	{ return this; }

	static int count;

	//	private:
	policies_test_class(policies_test_class const& c): name_(c.name_)
	{ ++count; }

	void member_out_val(int a, int* v) { *v = a * 2; }
	secret_type* member_secret() { return &sec_; }
};

int policies_test_class::count = 0;

policies_test_class global;

void out_val(float* f) { *f = 3.f; }
policies_test_class* copy_val() { return &global; }
policies_test_class const* copy_val_const() { return &global; }

secret_type* secret() { return &sec_; }

void aux_test();

struct test_t {
	test_t *make(int) { return new test_t(); }
	void take(test_t*) {}
};

struct MI2;

struct MI1
{
	void add(MI2 *) {}
};

struct MI2 : public MI1
{
	virtual ~MI2()
	{}
};

struct MI2W : public MI2, public luabind::wrap_base {};

void test_main(lua_State* L)
{
	using namespace luabind;

	module(L)
	[
		class_<test_t>("test_t")
		.def("make", &test_t::make, adopt(return_value))
		.def("take", &test_t::take, adopt(_2))
	];

	module(L)
	[
		class_<policies_test_class>("test")
			.def(constructor<>())
			.def("member_out_val", &policies_test_class::member_out_val, pure_out_value(_3))
			.def("member_secret", &policies_test_class::member_secret, discard_result)
			.def("f", &policies_test_class::f, adopt(_2))
			.def("make", &policies_test_class::make, adopt(return_value))
			.def("internal_ref", &policies_test_class::internal_ref, dependency(result, _1))
			.def("self_ref", &policies_test_class::self_ref, return_reference_to(_1)),

		def("out_val", &out_val, pure_out_value(_1)),
		def("copy_val", &copy_val, copy(result)),
		def("copy_val_const", &copy_val_const, copy(result)),
		def("secret", &secret, discard_result),

		class_<MI1>("mi1")
			.def(constructor<>())
			.def("add",&MI1::add,adopt(_2)),

		class_<MI2,MI2W,MI1>("mi2")
			.def(constructor<>())
	];

	// test copy
	DOSTRING(L,	"a = secret()\n");

	TEST_CHECK(policies_test_class::count == 1);

	DOSTRING(L, "a = copy_val()\n");
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "b = copy_val_const()\n");
	TEST_CHECK(policies_test_class::count == 3);

	DOSTRING(L,
		"a = nil\n"
		"b = nil\n"
		"collectgarbage()\n");

	// only the global variable left here
	TEST_CHECK(policies_test_class::count == 1);

	// out_value
	DOSTRING(L,
		"a = out_val()\n"
		"assert(a == 3)");

	// return_reference_to
	DOSTRING(L,
		"a = test()\n"
		"b = a:self_ref()\n"
		"a = nil\n"
		"collectgarbage()");

	// a is kept alive as long as b is alive
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L,
		"b = nil\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	DOSTRING(L, "a = test()");

	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L,
		"b = a:internal_ref()\n"
		"a = nil\n"
		"collectgarbage()");

	// a is kept alive as long as b is alive
	TEST_CHECK(policies_test_class::count == 2);

	// two gc-cycles because dependency-table won't be collected in the
	// same cycle as the object_rep
	DOSTRING(L,
		"b = nil\n"
		"collectgarbage()\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	// adopt
	DOSTRING(L, "a = test()");

	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "b = a:make('tjosan')");
	DOSTRING(L, "assert(a:member_out_val(3) == 6)");
	DOSTRING(L, "a:member_secret()");

	// make instantiated a new policies_test_class
	TEST_CHECK(policies_test_class::count == 3);

	DOSTRING(L, "a:f(b)\n");

	// b was adopted by c++ and deleted the object
	TEST_CHECK(policies_test_class::count == 2);

	DOSTRING(L, "a = nil\n"
		"collectgarbage()");

	TEST_CHECK(policies_test_class::count == 1);

	// adopt with wrappers
	DOSTRING(L, "mi1():add(mi2())");
}

<|endoftext|>


<commit_before>//#include "OpenCLHelper.h"
//#include "ClConvolve.h"

#include <iostream>
#ifdef MPI_AVAILABLE
#include "mpi.h"
#endif

#include "BoardHelper.h"
#include "MnistLoader.h"
#include "BoardPng.h"
#include "Timer.h"
#include "NeuralNet.h"
#include "AccuracyHelper.h"
#include "stringhelper.h"
#include "FileHelper.h"
#include "StatefulTimer.h"
#include "WeightsPersister.h"

using namespace std;

int myrank = 0;
int mysize = 1;

void loadMnist( string mnistDir, string setName, int *p_N, int *p_boardSize, float ****p_images, int **p_labels ) {
    int boardSize;
    int Nboards;
    int Nlabels;
    // images
    int ***boards = MnistLoader::loadImages( mnistDir, setName, &Nboards, &boardSize );
    int *labels = MnistLoader::loadLabels( mnistDir, setName, &Nlabels );
    if( Nboards != Nlabels ) {
         throw runtime_error("mismatch between number of boards, and number of labels " + toString(Nboards ) + " vs " +
             toString(Nlabels ) );
    }
    if( myrank == 0 ) cout << "loaded " << Nboards << " boards.  " << endl;
//    MnistLoader::shuffle( boards, labels, Nboards, boardSize );
    float ***boardsFloat = BoardsHelper::allocateBoardsFloats( Nboards, boardSize );
    BoardsHelper::copyBoards( boardsFloat, boards, Nboards, boardSize );
    BoardsHelper::deleteBoards( &boards, Nboards, boardSize );
    *p_images = boardsFloat;
    *p_labels = labels;

    *p_boardSize = boardSize;
    *p_N = Nboards;
}

void getStats( float ***boards, int N, int boardSize, float *p_mean, float *p_thismax ) {
    // get mean of the dataset
    int count = 0;
    float thismax = 0;
   float sum = 0;
    for( int n = 0; n < N; n++ ) {
       for( int i = 0; i < boardSize; i++ ) {
          for( int j = 0; j < boardSize; j++ ) {
              count++;
              sum += boards[n][i][j];
              thismax = max( thismax, boards[n][i][j] );
          }
       }
    }
    *p_mean = sum / count;
    *p_thismax = thismax;
}

void normalize( float ***boards, int N, int boardSize, double mean, double thismax ) {
    for( int n = 0; n < N; n++ ) {
       for( int i = 0; i < boardSize; i++ ) {
          for( int j = 0; j < boardSize; j++ ) {
              boards[n][i][j] = boards[n][i][j] / thismax - 0.1;
          }
       }
    }
}

class Config {
public:
    string dataDir = "../data/mnist";
    string trainSet = "train";
    string testSet = "t10k";
    int numTrain = 60000;
    int numTest = 10000;
    int batchSize = 128;
    int numEpochs = 20;
    int numFilters = 16;
    int numLayers = 1;
    int padZeros = 0;
    int filterSize = 5;
    int restartable = 0;
    string restartableFilename = "weights.dat";
    float learningRate = 0.0001f;
    int biased = 1;
    string resultsFilename = "results.txt";
    Config() {
    }
};

float printAccuracy( string name, NeuralNet *net, float ***boards, int *labels, int batchSize, int N ) {
    int testNumRight = 0;
    net->setBatchSize( batchSize );
    int numBatches = (N + batchSize - 1 ) / batchSize;
    for( int batch = 0; batch < numBatches; batch++ ) {
        int batchStart = batch * batchSize;
        int thisBatchSize = batchSize;
        if( batch == numBatches - 1 ) {
            thisBatchSize = N - batchStart;
            net->setBatchSize( thisBatchSize );
        }
        net->propagate( &(boards[batchStart][0][0]) );
        float const*results = net->getResults();
        int thisnumright = net->calcNumRight( &(labels[batchStart]) );
        testNumRight += thisnumright;
    }
    float accuracy = ( testNumRight * 100.0f / N );
    if( myrank == 0 ) cout << name << " overall: " << testNumRight << "/" << N << " " << accuracy << "%" << endl;
    return accuracy;
}

void go(Config config) {
    Timer timer;

    int boardSize;

    float ***boardsFloat = 0;
    int *labels = 0;

    float ***boardsTest = 0;
    int *labelsTest = 0;
    {
        int N;
        loadMnist( config.dataDir, config.trainSet, &N, &boardSize, &boardsFloat, &labels );

        int Ntest;
        loadMnist( config.dataDir, config.testSet, &Ntest, &boardSize, &boardsTest, &labelsTest );

    }

    float mean;
    float thismax;
//    getStats( boardsFloat, config.numTrain, boardSize, &mean, &thismax );
    mean = 33;
    thismax = 255;
    if( myrank == 0 ) cout << " board stats mean " << mean << " max " << thismax << " boardSize " << boardSize << endl;
    normalize( boardsFloat, config.numTrain, boardSize, mean, thismax );
    normalize( boardsTest, config.numTest, boardSize, mean, thismax );
    if( myrank == 0 ) timer.timeCheck("after load images");

    int numToTrain = config.numTrain;
    const int batchSize = config.batchSize;
    NeuralNet *net = NeuralNet::maker()->planes(1)->boardSize(boardSize)->instance();
    for( int i = 0; i < config.numLayers; i++ ) {
        net->convolutionalMaker()->numFilters(config.numFilters)->filterSize(config.filterSize)->relu()->biased()->padZeros(config.padZeros)->insert();
    }
    net->fullyConnectedMaker()->numPlanes(10)->boardSize(1)->linear()->biased(config.biased)->insert();
    net->softMaxLossMaker()->insert();
    net->setBatchSize(config.batchSize);
    net->print();

    if( config.restartable ) {
        WeightsPersister::loadWeights( config.restartableFilename, net );
    }

    timer.timeCheck("before learning start");
    StatefulTimer::timeCheck("START");
    const int totalWeightsSize = WeightsPersister::getTotalNumWeights(net);
    float *weightsCopy = new float[totalWeightsSize];
    float *newWeights = new float[totalWeightsSize];
    float *weightsChange = new float[totalWeightsSize];
    float *weightsChangeReduced = new float[totalWeightsSize];
    for( int epoch = 0; epoch < config.numEpochs; epoch++ ) {
        int trainTotalNumber = 0;
        int trainNumRight = 0;
        int numBatches = ( config.numTrain + config.batchSize - 1 ) / config.batchSize;
        int eachNodeBatchSize = config.batchSize / mysize;
        int thisNodeBatchSize = eachNodeBatchSize;
        if( myrank == mysize - 1 ) {
            thisNodeBatchSize = config.batchSize - ( mysize - 1 ) * eachNodeBatchSize;
        }
        net->setBatchSize( thisNodeBatchSize );
        float loss = 0;
        for( int batch = 0; batch < numBatches; batch++ ) {
            int batchStart = batch * config.batchSize;
            int thisBatchSize = config.batchSize;
            int nodeBatchStart = batchStart + myrank * eachNodeBatchSize;
            if( batch == numBatches - 1 ) {
                thisBatchSize = config.numTrain - batchStart;
                eachNodeBatchSize = thisBatchSize / mysize;
                nodeBatchStart = batchStart + myrank * eachNodeBatchSize;
                thisNodeBatchSize = eachNodeBatchSize;
                if( myrank == mysize - 1 ) {
                    thisNodeBatchSize = thisBatchSize - ( mysize - 1 ) * eachNodeBatchSize;
                }
                net->setBatchSize( thisNodeBatchSize );
            }
            #ifdef MPI_AVAILABLE
            WeightsPersister::copyNetWeightsToArray( net, weightsCopy );
            #endif
            net->propagate( &(boardsFloat[nodeBatchStart][0][0]) );
            net->backPropFromLabels( config.learningRate, &(labels[nodeBatchStart]) );
            trainTotalNumber += thisNodeBatchSize;
            trainNumRight += net->calcNumRight( &(labels[nodeBatchStart]) );
            loss += net->calcLossFromLabels( &(labels[nodeBatchStart]) );
            // share out the weights... just average them?
            // for each weight, wnew = wold + dw
            // if multiple changes, wnew = wold + dw1 + dw2 + dw3 + ...
            // if each node doing it, then wnew1 = wold + dw1; wnew2 = wold + dw2
            // wnew1 + wnew2 = wold * 2 + dw1 + dw2
            // we want: wnew = wold + dw1 + dw2 = wnew1 + wnew2 - wold
            // seems like we should keep a copy of the old weights, otherwise cannot compute
            #ifdef MPI_AVAILABLE
            WeightsPersister::copyNetWeightsToArray( net, newWeights );
            if( myrank == 0 ) {
                for( int i = 0; i < totalWeightsSize; i++ ) {
                    weightsChange[i] = newWeights[i];
                }
            } else {
                for( int i = 0; i < totalWeightsSize; i++ ) {
                    weightsChange[i] = newWeights[i] - weightsCopy[i];
                }
            }
            MPI_Allreduce( weightsChange, weightsChangeReduced, totalWeightsSize, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD );
            WeightsPersister::copyArrayToNetWeights( weightsChangeReduced, net );
            StatefulTimer::timeCheck("allreduced weights");
            #endif
        }
        StatefulTimer::dump(true);
        if( myrank == 0 ) cout << "       loss L: " << loss << endl;
        if( myrank == 0 ) timer.timeCheck("after epoch " + toString(epoch) );
//        net->print();
        if( myrank == 0 ) std::cout << "train accuracy: " << trainNumRight << "/" << trainTotalNumber << " " << (trainNumRight * 100.0f/ trainTotalNumber) << "%" << std::endl;
        if( myrank == 0 ) printAccuracy( "test", net, boardsTest, labelsTest, batchSize, config.numTest );
        if( myrank == 0 ) timer.timeCheck("after tests");
        if( config.restartable ) {
            WeightsPersister::persistWeights( config.restartableFilename, net );
        }
    }
    delete[] weightsCopy;

    if( myrank == 0 ) printAccuracy( "test", net, boardsTest, labelsTest, batchSize, config.numTest );
    if( myrank == 0 ) timer.timeCheck("after tests");

    int numTestBatches = ( config.numTest + config.batchSize - 1 ) / config.batchSize;
    int totalNumber = 0;
    int totalNumRight = 0;
    net->setBatchSize( config.batchSize );
    for( int batch = 0; batch < numTestBatches; batch++ ) {
        int batchStart = batch * config.batchSize;
        int thisBatchSize = config.batchSize;
        if( batch == numTestBatches - 1 ) {
            thisBatchSize = config.numTest - batchStart;
            net->setBatchSize( thisBatchSize );
        }
        net->propagate( &(boardsTest[batchStart][0][0]) );
        float const*resultsTest = net->getResults();
        totalNumber += thisBatchSize;
        totalNumRight += net->calcNumRight( &(labelsTest[batchStart]) );
    }
    if( myrank == 0 ) cout << "test accuracy : " << totalNumRight << "/" << totalNumber << endl;

    delete net;

    delete[] labelsTest;
//    BoardsHelper::deleteBoards( &boardsTest, Ntest, boardSize );

    delete[] labels;
//    BoardsHelper::deleteBoards( &boardsFloat, N, boardSize );
}

int main( int argc, char *argv[] ) {
    #ifdef MPI_AVAILABLE
        MPI_Init( &argc, &argv );
        MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
        MPI_Comm_size( MPI_COMM_WORLD, &mysize );
    #endif

    Config config;
    if( myrank == 0 ) {
        if( argc == 2 && ( string(argv[1]) == "--help" || string(argv[1]) == "--?" || string(argv[1]) == "-?" || string(argv[1]) == "-h" ) ) {
            cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
            cout << "Possible key=value pairs:" << endl;
            cout << "    datadir=[data directory] (" << config.dataDir << ")" << endl;
            cout << "    trainset=[train|t10k|other set name] (" << config.trainSet << ")" << endl;
            cout << "    testset=[train|t10k|other set name] (" << config.testSet << ")" << endl;
            cout << "    numtrain=[num training examples] (" << config.numTrain << ")" << endl;
            cout << "    numtest=[num test examples] (" << config.numTest << ")" << endl;
            cout << "    batchsize=[batch size] (" << config.batchSize << ")" << endl;
            cout << "    numepochs=[number epochs] (" << config.numEpochs << ")" << endl;
            cout << "    numlayers=[number convolutional layers] (" << config.numLayers << ")" << endl;
            cout << "    numfilters=[number filters] (" << config.numFilters << ")" << endl;
            cout << "    filtersize=[filter size] (" << config.filterSize << ")" << endl;
            cout << "    biased=[0|1] (" << config.biased << ")" << endl;
            cout << "    padzeros=[0|1] (" << config.padZeros << ")" << endl;
            cout << "    learningrate=[learning rate, a float value] (" << config.learningRate << ")" << endl;
            cout << "    restartable=[weights are persistent?] (" << config.restartable << ")" << endl;
            cout << "    restartablefilename=[filename to store weights] (" << config.restartableFilename << ")" << endl;
            cout << "    resultsfilename=[filename to store results] (" << config.resultsFilename << ")" << endl;
        }
    }
    for( int i = 1; i < argc; i++ ) {
       vector<string> splitkeyval = split( argv[i], "=" );
       if( splitkeyval.size() != 2 ) {
            if( myrank == 0  ){
                cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
            }
            #ifdef MPI_AVAILABLE
            MPI_Finalize();
            #endif
            exit(1);
       } else {
           string key = splitkeyval[0];
           string value = splitkeyval[1];
           if( key == "datadir" ) config.dataDir = value;
           if( key == "trainset" ) config.trainSet = value;
           if( key == "testset" ) config.testSet = value;
           if( key == "numtrain" ) config.numTrain = atoi(value);
           if( key == "numtest" ) config.numTest = atoi(value);
           if( key == "batchsize" ) config.batchSize = atoi(value);
           if( key == "numepochs" ) config.numEpochs = atoi(value);
           if( key == "biased" ) config.biased = atoi(value);
           if( key == "numfilters" ) config.numFilters = atoi(value);
           if( key == "numlayers" ) config.numLayers = atoi(value);
           if( key == "padzeros" ) config.padZeros = atoi(value);
           if( key == "filtersize" ) config.filterSize = atoi(value);
           if( key == "learningrate" ) config.learningRate = atof(value);
           if( key == "restartable" ) config.restartable = atoi(value);
           if( key == "restartablefilename" ) config.restartableFilename = value;
           if( key == "resultsfilename" ) config.resultsFilename = value;
       }
    }
    go( config );
    #ifdef MPI_AVAILABLE
    MPI_Finalize();
    #endif
}

<commit_msg>improve timing printout for mpit a bit<commit_after>//#include "OpenCLHelper.h"
//#include "ClConvolve.h"

#include <iostream>
#ifdef MPI_AVAILABLE
#include "mpi.h"
#endif

#include "BoardHelper.h"
#include "MnistLoader.h"
#include "BoardPng.h"
#include "Timer.h"
#include "NeuralNet.h"
#include "AccuracyHelper.h"
#include "stringhelper.h"
#include "FileHelper.h"
#include "StatefulTimer.h"
#include "WeightsPersister.h"

using namespace std;

int myrank = 0;
int mysize = 1;

void loadMnist( string mnistDir, string setName, int *p_N, int *p_boardSize, float ****p_images, int **p_labels ) {
    int boardSize;
    int Nboards;
    int Nlabels;
    // images
    int ***boards = MnistLoader::loadImages( mnistDir, setName, &Nboards, &boardSize );
    int *labels = MnistLoader::loadLabels( mnistDir, setName, &Nlabels );
    if( Nboards != Nlabels ) {
         throw runtime_error("mismatch between number of boards, and number of labels " + toString(Nboards ) + " vs " +
             toString(Nlabels ) );
    }
    if( myrank == 0 ) cout << "loaded " << Nboards << " boards.  " << endl;
//    MnistLoader::shuffle( boards, labels, Nboards, boardSize );
    float ***boardsFloat = BoardsHelper::allocateBoardsFloats( Nboards, boardSize );
    BoardsHelper::copyBoards( boardsFloat, boards, Nboards, boardSize );
    BoardsHelper::deleteBoards( &boards, Nboards, boardSize );
    *p_images = boardsFloat;
    *p_labels = labels;

    *p_boardSize = boardSize;
    *p_N = Nboards;
}

void getStats( float ***boards, int N, int boardSize, float *p_mean, float *p_thismax ) {
    // get mean of the dataset
    int count = 0;
    float thismax = 0;
   float sum = 0;
    for( int n = 0; n < N; n++ ) {
       for( int i = 0; i < boardSize; i++ ) {
          for( int j = 0; j < boardSize; j++ ) {
              count++;
              sum += boards[n][i][j];
              thismax = max( thismax, boards[n][i][j] );
          }
       }
    }
    *p_mean = sum / count;
    *p_thismax = thismax;
}

void normalize( float ***boards, int N, int boardSize, double mean, double thismax ) {
    for( int n = 0; n < N; n++ ) {
       for( int i = 0; i < boardSize; i++ ) {
          for( int j = 0; j < boardSize; j++ ) {
              boards[n][i][j] = boards[n][i][j] / thismax - 0.1;
          }
       }
    }
}

class Config {
public:
    string dataDir = "../data/mnist";
    string trainSet = "train";
    string testSet = "t10k";
    int numTrain = 60000;
    int numTest = 10000;
    int batchSize = 128;
    int numEpochs = 20;
    int numFilters = 16;
    int numLayers = 1;
    int padZeros = 0;
    int filterSize = 5;
    int restartable = 0;
    string restartableFilename = "weights.dat";
    float learningRate = 0.0001f;
    int biased = 1;
    string resultsFilename = "results.txt";
    Config() {
    }
};

float printAccuracy( string name, NeuralNet *net, float ***boards, int *labels, int batchSize, int N ) {
    int testNumRight = 0;
    net->setBatchSize( batchSize );
    int numBatches = (N + batchSize - 1 ) / batchSize;
    for( int batch = 0; batch < numBatches; batch++ ) {
        int batchStart = batch * batchSize;
        int thisBatchSize = batchSize;
        if( batch == numBatches - 1 ) {
            thisBatchSize = N - batchStart;
            net->setBatchSize( thisBatchSize );
        }
        net->propagate( &(boards[batchStart][0][0]) );
        float const*results = net->getResults();
        int thisnumright = net->calcNumRight( &(labels[batchStart]) );
        testNumRight += thisnumright;
    }
    float accuracy = ( testNumRight * 100.0f / N );
    if( myrank == 0 ) cout << name << " overall: " << testNumRight << "/" << N << " " << accuracy << "%" << endl;
    return accuracy;
}

void go(Config config) {
    Timer timer;

    int boardSize;

    float ***boardsFloat = 0;
    int *labels = 0;

    float ***boardsTest = 0;
    int *labelsTest = 0;
    {
        int N;
        loadMnist( config.dataDir, config.trainSet, &N, &boardSize, &boardsFloat, &labels );

        int Ntest;
        loadMnist( config.dataDir, config.testSet, &Ntest, &boardSize, &boardsTest, &labelsTest );

    }

    float mean;
    float thismax;
//    getStats( boardsFloat, config.numTrain, boardSize, &mean, &thismax );
    mean = 33;
    thismax = 255;
    if( myrank == 0 ) cout << " board stats mean " << mean << " max " << thismax << " boardSize " << boardSize << endl;
    normalize( boardsFloat, config.numTrain, boardSize, mean, thismax );
    normalize( boardsTest, config.numTest, boardSize, mean, thismax );
    if( myrank == 0 ) timer.timeCheck("after load images");

    int numToTrain = config.numTrain;
    const int batchSize = config.batchSize;
    NeuralNet *net = NeuralNet::maker()->planes(1)->boardSize(boardSize)->instance();
    for( int i = 0; i < config.numLayers; i++ ) {
        net->convolutionalMaker()->numFilters(config.numFilters)->filterSize(config.filterSize)->relu()->biased()->padZeros(config.padZeros)->insert();
    }
    net->fullyConnectedMaker()->numPlanes(10)->boardSize(1)->linear()->biased(config.biased)->insert();
    net->softMaxLossMaker()->insert();
    net->setBatchSize(config.batchSize);
    net->print();

    if( config.restartable ) {
        WeightsPersister::loadWeights( config.restartableFilename, net );
    }

    timer.timeCheck("before learning start");
    StatefulTimer::timeCheck("START");
    const int totalWeightsSize = WeightsPersister::getTotalNumWeights(net);
    cout << "totalweightssize: " << totalWeightsSize << endl;
    float *weightsCopy = new float[totalWeightsSize];
    float *newWeights = new float[totalWeightsSize];
    float *weightsChange = new float[totalWeightsSize];
    float *weightsChangeReduced = new float[totalWeightsSize];
    for( int epoch = 0; epoch < config.numEpochs; epoch++ ) {
        int trainTotalNumber = 0;
        int trainNumRight = 0;
        int numBatches = ( config.numTrain + config.batchSize - 1 ) / config.batchSize;
        int eachNodeBatchSize = config.batchSize / mysize;
        int thisNodeBatchSize = eachNodeBatchSize;
        if( myrank == mysize - 1 ) {
            thisNodeBatchSize = config.batchSize - ( mysize - 1 ) * eachNodeBatchSize;
        }
        net->setBatchSize( thisNodeBatchSize );
        float loss = 0;
        for( int batch = 0; batch < numBatches; batch++ ) {
            int batchStart = batch * config.batchSize;
            int thisBatchSize = config.batchSize;
            int nodeBatchStart = batchStart + myrank * eachNodeBatchSize;
            if( batch == numBatches - 1 ) {
                thisBatchSize = config.numTrain - batchStart;
                eachNodeBatchSize = thisBatchSize / mysize;
                nodeBatchStart = batchStart + myrank * eachNodeBatchSize;
                thisNodeBatchSize = eachNodeBatchSize;
                if( myrank == mysize - 1 ) {
                    thisNodeBatchSize = thisBatchSize - ( mysize - 1 ) * eachNodeBatchSize;
                }
                net->setBatchSize( thisNodeBatchSize );
            }
            #ifdef MPI_AVAILABLE
            StatefulTimer::timeCheck("copyNetWeightsToArray START");
            WeightsPersister::copyNetWeightsToArray( net, weightsCopy );
            StatefulTimer::timeCheck("copyNetWeightsToArray END");
            #endif
            net->propagate( &(boardsFloat[nodeBatchStart][0][0]) );
            net->backPropFromLabels( config.learningRate, &(labels[nodeBatchStart]) );
            trainTotalNumber += thisNodeBatchSize;
            trainNumRight += net->calcNumRight( &(labels[nodeBatchStart]) );
            loss += net->calcLossFromLabels( &(labels[nodeBatchStart]) );
            // share out the weights... just average them?
            // for each weight, wnew = wold + dw
            // if multiple changes, wnew = wold + dw1 + dw2 + dw3 + ...
            // if each node doing it, then wnew1 = wold + dw1; wnew2 = wold + dw2
            // wnew1 + wnew2 = wold * 2 + dw1 + dw2
            // we want: wnew = wold + dw1 + dw2 = wnew1 + wnew2 - wold
            // seems like we should keep a copy of the old weights, otherwise cannot compute
            #ifdef MPI_AVAILABLE
            StatefulTimer::timeCheck("allreduce START");
            WeightsPersister::copyNetWeightsToArray( net, newWeights );
            if( myrank == 0 ) {
                for( int i = 0; i < totalWeightsSize; i++ ) {
                    weightsChange[i] = newWeights[i];
                }
            } else {
                for( int i = 0; i < totalWeightsSize; i++ ) {
                    weightsChange[i] = newWeights[i] - weightsCopy[i];
                }
            }
            MPI_Allreduce( weightsChange, weightsChangeReduced, totalWeightsSize, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD );
            WeightsPersister::copyArrayToNetWeights( weightsChangeReduced, net );
            StatefulTimer::timeCheck("allreduce END");
            #endif
        }
        StatefulTimer::dump(true);
        if( myrank == 0 ) cout << "       loss L: " << loss << endl;
        if( myrank == 0 ) timer.timeCheck("after epoch " + toString(epoch) );
//        net->print();
        if( myrank == 0 ) std::cout << "train accuracy: " << trainNumRight << "/" << trainTotalNumber << " " << (trainNumRight * 100.0f/ trainTotalNumber) << "%" << std::endl;
        if( myrank == 0 ) printAccuracy( "test", net, boardsTest, labelsTest, batchSize, config.numTest );
        if( myrank == 0 ) timer.timeCheck("after tests");
        if( config.restartable ) {
            WeightsPersister::persistWeights( config.restartableFilename, net );
        }
    }
    delete[] weightsCopy;

    if( myrank == 0 ) printAccuracy( "test", net, boardsTest, labelsTest, batchSize, config.numTest );
    if( myrank == 0 ) timer.timeCheck("after tests");

    int numTestBatches = ( config.numTest + config.batchSize - 1 ) / config.batchSize;
    int totalNumber = 0;
    int totalNumRight = 0;
    net->setBatchSize( config.batchSize );
    for( int batch = 0; batch < numTestBatches; batch++ ) {
        int batchStart = batch * config.batchSize;
        int thisBatchSize = config.batchSize;
        if( batch == numTestBatches - 1 ) {
            thisBatchSize = config.numTest - batchStart;
            net->setBatchSize( thisBatchSize );
        }
        net->propagate( &(boardsTest[batchStart][0][0]) );
        float const*resultsTest = net->getResults();
        totalNumber += thisBatchSize;
        totalNumRight += net->calcNumRight( &(labelsTest[batchStart]) );
    }
    if( myrank == 0 ) cout << "test accuracy : " << totalNumRight << "/" << totalNumber << endl;

    delete net;

    delete[] labelsTest;
//    BoardsHelper::deleteBoards( &boardsTest, Ntest, boardSize );

    delete[] labels;
//    BoardsHelper::deleteBoards( &boardsFloat, N, boardSize );
}

int main( int argc, char *argv[] ) {
    #ifdef MPI_AVAILABLE
        MPI_Init( &argc, &argv );
        MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
        MPI_Comm_size( MPI_COMM_WORLD, &mysize );
    #endif

    Config config;
    if( myrank == 0 ) {
        if( argc == 2 && ( string(argv[1]) == "--help" || string(argv[1]) == "--?" || string(argv[1]) == "-?" || string(argv[1]) == "-h" ) ) {
            cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
            cout << "Possible key=value pairs:" << endl;
            cout << "    datadir=[data directory] (" << config.dataDir << ")" << endl;
            cout << "    trainset=[train|t10k|other set name] (" << config.trainSet << ")" << endl;
            cout << "    testset=[train|t10k|other set name] (" << config.testSet << ")" << endl;
            cout << "    numtrain=[num training examples] (" << config.numTrain << ")" << endl;
            cout << "    numtest=[num test examples] (" << config.numTest << ")" << endl;
            cout << "    batchsize=[batch size] (" << config.batchSize << ")" << endl;
            cout << "    numepochs=[number epochs] (" << config.numEpochs << ")" << endl;
            cout << "    numlayers=[number convolutional layers] (" << config.numLayers << ")" << endl;
            cout << "    numfilters=[number filters] (" << config.numFilters << ")" << endl;
            cout << "    filtersize=[filter size] (" << config.filterSize << ")" << endl;
            cout << "    biased=[0|1] (" << config.biased << ")" << endl;
            cout << "    padzeros=[0|1] (" << config.padZeros << ")" << endl;
            cout << "    learningrate=[learning rate, a float value] (" << config.learningRate << ")" << endl;
            cout << "    restartable=[weights are persistent?] (" << config.restartable << ")" << endl;
            cout << "    restartablefilename=[filename to store weights] (" << config.restartableFilename << ")" << endl;
            cout << "    resultsfilename=[filename to store results] (" << config.resultsFilename << ")" << endl;
        }
    }
    for( int i = 1; i < argc; i++ ) {
       vector<string> splitkeyval = split( argv[i], "=" );
       if( splitkeyval.size() != 2 ) {
            if( myrank == 0  ){
                cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
            }
            #ifdef MPI_AVAILABLE
            MPI_Finalize();
            #endif
            exit(1);
       } else {
           string key = splitkeyval[0];
           string value = splitkeyval[1];
           if( key == "datadir" ) config.dataDir = value;
           if( key == "trainset" ) config.trainSet = value;
           if( key == "testset" ) config.testSet = value;
           if( key == "numtrain" ) config.numTrain = atoi(value);
           if( key == "numtest" ) config.numTest = atoi(value);
           if( key == "batchsize" ) config.batchSize = atoi(value);
           if( key == "numepochs" ) config.numEpochs = atoi(value);
           if( key == "biased" ) config.biased = atoi(value);
           if( key == "numfilters" ) config.numFilters = atoi(value);
           if( key == "numlayers" ) config.numLayers = atoi(value);
           if( key == "padzeros" ) config.padZeros = atoi(value);
           if( key == "filtersize" ) config.filterSize = atoi(value);
           if( key == "learningrate" ) config.learningRate = atof(value);
           if( key == "restartable" ) config.restartable = atoi(value);
           if( key == "restartablefilename" ) config.restartableFilename = value;
           if( key == "resultsfilename" ) config.resultsFilename = value;
       }
    }
    go( config );
    #ifdef MPI_AVAILABLE
    MPI_Finalize();
    #endif
}

<|endoftext|>


<commit_before>// Unit Tests for Scintilla internal data structures

/*
    Currently tested:
        SplitVector
        Partitioning
        ContractionState

    To do:
        RunStyles
        Decoration
        DecorationList
        PerLine *
        CellBuffer *
        Range
        StyledText
        CaseFolder ...
        Document
        RESearch
        Selection
        UniConversion
        Style

        lexlib:
        Accessor
        LexAccessor
        CharacterSet
        OptionSet
        PropSetSimple
        StyleContext
        WordList
*/

#include <stdio.h>

#include "Platform.h"

#include <gtest/gtest.h>

// Needed for PLATFORM_ASSERT in code being tested

void Platform::Assert(const char *c, const char *file, int line) {
	fprintf(stderr, "Assertion [%s] failed at %s %d\n", c, file, line);
	abort();
}

int main(int argc, char **argv) {
	testing::InitGoogleTest(&argc, argv);
	return RUN_ALL_TESTS();
}
<commit_msg>Added runStyles tests.<commit_after>// Unit Tests for Scintilla internal data structures

/*
    Currently tested:
        SplitVector
        Partitioning
        RunStyles
        ContractionState

    To do:
        Decoration
        DecorationList
        PerLine *
        CellBuffer *
        Range
        StyledText
        CaseFolder ...
        Document
        RESearch
        Selection
        UniConversion
        Style

        lexlib:
        Accessor
        LexAccessor
        CharacterSet
        OptionSet
        PropSetSimple
        StyleContext
        WordList
*/

#include <stdio.h>

#include "Platform.h"

#include <gtest/gtest.h>

// Needed for PLATFORM_ASSERT in code being tested

void Platform::Assert(const char *c, const char *file, int line) {
	fprintf(stderr, "Assertion [%s] failed at %s %d\n", c, file, line);
	abort();
}

int main(int argc, char **argv) {
	testing::InitGoogleTest(&argc, argv);
	return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>/*=============================================================================

  Library: CppMicroServices

  Copyright (c) German Cancer Research Center,
    Division of Medical and Biological Informatics

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

=============================================================================*/

#include "usTestManager.h"

#include "usModuleImport.h"

US_BEGIN_NAMESPACE

TestManager& TestManager::GetInstance()
{
  static TestManager instance;
  return instance;
}

void TestManager::Initialize()
{
  m_FailedTests = 0;
  m_PassedTests = 0;
}

int TestManager::NumberOfFailedTests()
{
  return m_FailedTests;
}

int TestManager::NumberOfPassedTests()
{
  return m_PassedTests;
}

void TestManager::TestFailed()
{
  m_FailedTests++;
}

void TestManager::TestPassed()
{
  m_PassedTests++;
}

US_END_NAMESPACE

#ifndef US_BUILD_SHARED_LIBS
US_IMPORT_MODULE_RESOURCES(CppMicroServices)
US_IMPORT_MODULE(TestModuleA)
US_IMPORT_MODULE(TestModuleA2)
US_IMPORT_MODULE(TestModuleB)
US_IMPORT_MODULE_RESOURCES(TestModuleB)
US_IMPORT_MODULE(TestModuleM)
US_IMPORT_MODULE_RESOURCES(TestModuleM)
US_IMPORT_MODULE(TestModuleR)
US_IMPORT_MODULE_RESOURCES(TestModuleR)
US_IMPORT_MODULE(TestModuleS)
US_IMPORT_MODULE(TestModuleSL1)
US_IMPORT_MODULE(TestModuleSL3)
US_IMPORT_MODULE(TestModuleSL4)

US_LOAD_IMPORTED_MODULES_INTO_MAIN(
    TestModuleA TestModuleA2 TestModuleB TestModuleImportedByB TestModuleM
    TestModuleR TestModuleS TestModuleSL1 TestModuleSL3 TestModuleSL4
    )
#endif
<commit_msg>Import TestModuleH for static unit tests.<commit_after>/*=============================================================================

  Library: CppMicroServices

  Copyright (c) German Cancer Research Center,
    Division of Medical and Biological Informatics

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

=============================================================================*/

#include "usTestManager.h"

#include "usModuleImport.h"

US_BEGIN_NAMESPACE

TestManager& TestManager::GetInstance()
{
  static TestManager instance;
  return instance;
}

void TestManager::Initialize()
{
  m_FailedTests = 0;
  m_PassedTests = 0;
}

int TestManager::NumberOfFailedTests()
{
  return m_FailedTests;
}

int TestManager::NumberOfPassedTests()
{
  return m_PassedTests;
}

void TestManager::TestFailed()
{
  m_FailedTests++;
}

void TestManager::TestPassed()
{
  m_PassedTests++;
}

US_END_NAMESPACE

#ifndef US_BUILD_SHARED_LIBS
US_IMPORT_MODULE_RESOURCES(CppMicroServices)
US_IMPORT_MODULE(TestModuleA)
US_IMPORT_MODULE(TestModuleA2)
US_IMPORT_MODULE(TestModuleB)
US_IMPORT_MODULE_RESOURCES(TestModuleB)
US_IMPORT_MODULE(TestModuleH)
US_IMPORT_MODULE(TestModuleM)
US_IMPORT_MODULE_RESOURCES(TestModuleM)
US_IMPORT_MODULE(TestModuleR)
US_IMPORT_MODULE_RESOURCES(TestModuleR)
US_IMPORT_MODULE(TestModuleS)
US_IMPORT_MODULE(TestModuleSL1)
US_IMPORT_MODULE(TestModuleSL3)
US_IMPORT_MODULE(TestModuleSL4)

US_LOAD_IMPORTED_MODULES_INTO_MAIN(
    TestModuleA TestModuleA2 TestModuleB TestModuleImportedByB TestModuleH TestModuleM
    TestModuleR TestModuleS TestModuleSL1 TestModuleSL3 TestModuleSL4
    )
#endif
<|endoftext|>


<commit_before>/***********************************************************************
 *    filename:   NamedElement.cpp
 *    created:    30/10/2011
 *    author:     Martin Preisler
 *************************************************************************/
/***************************************************************************
 *   Copyright (C) 2004 - 2011 Paul D Turner & The CEGUI Development Team
 *
 *   Permission is hereby granted, free of charge, to any person obtaining
 *   a copy of this software and associated documentation files (the
 *   "Software"), to deal in the Software without restriction, including
 *   without limitation the rights to use, copy, modify, merge, publish,
 *   distribute, sublicense, and/or sell copies of the Software, and to
 *   permit persons to whom the Software is furnished to do so, subject to
 *   the following conditions:
 *
 *   The above copyright notice and this permission notice shall be
 *   included in all copies or substantial portions of the Software.
 *
 *   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 *   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 *   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 *   IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 *   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 *   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 *   OTHER DEALINGS IN THE SOFTWARE.
 ***************************************************************************/

#include "CEGUI/NamedElement.h"

#include <boost/test/unit_test.hpp>

BOOST_AUTO_TEST_SUITE(NamedElement)

BOOST_AUTO_TEST_CASE(Children)
{
    CEGUI::NamedElement* root = new CEGUI::NamedElement("root");
    BOOST_CHECK_EQUAL(root->getName(), "root");
    BOOST_CHECK_EQUAL(root->getProperty("Name"), "root");
    CEGUI::NamedElement* child1 = new CEGUI::NamedElement("child");
    CEGUI::NamedElement* child2 = new CEGUI::NamedElement("child");
    root->addChild(child1);
    BOOST_CHECK_THROW(root->addChild(child2), CEGUI::AlreadyExistsException);
    root->removeChild(child1);
    BOOST_CHECK_THROW(root->removeChild("child"), CEGUI::UnknownObjectException);
    root->addChild(child2);
    BOOST_CHECK_THROW(root->addChild(child1), CEGUI::AlreadyExistsException);

    delete child2;
    delete child1;
    delete root;
}

BOOST_AUTO_TEST_CASE(NamePath)
{
    CEGUI::NamedElement* root = new CEGUI::NamedElement("root");
    CEGUI::NamedElement* child = new CEGUI::NamedElement("child");
    CEGUI::NamedElement* inner_child = new CEGUI::NamedElement("inner_child");
    root->addChild(child);
    child->addChild(inner_child);

    BOOST_CHECK_EQUAL(child->getNamePath(), "root/child");
    BOOST_CHECK_EQUAL(child->getProperty("NamePath"), "root/child");
    BOOST_CHECK_EQUAL(inner_child->getNamePath(), "root/child/inner_child");

    BOOST_CHECK_EQUAL(root->getChildElement("child"), child);
    BOOST_CHECK_EQUAL(root->getChildElement("child/inner_child"), inner_child);
    BOOST_CHECK_EQUAL(child->getChildElement("inner_child"), inner_child);
    BOOST_CHECK_THROW(child->getChildElement("nonexistant"), CEGUI::UnknownObjectException);

    delete inner_child;
    delete child;
    delete root;
}

BOOST_AUTO_TEST_SUITE_END()
<commit_msg>Test string removeChild in NamedElement tests<commit_after>/***********************************************************************
 *    filename:   NamedElement.cpp
 *    created:    30/10/2011
 *    author:     Martin Preisler
 *************************************************************************/
/***************************************************************************
 *   Copyright (C) 2004 - 2011 Paul D Turner & The CEGUI Development Team
 *
 *   Permission is hereby granted, free of charge, to any person obtaining
 *   a copy of this software and associated documentation files (the
 *   "Software"), to deal in the Software without restriction, including
 *   without limitation the rights to use, copy, modify, merge, publish,
 *   distribute, sublicense, and/or sell copies of the Software, and to
 *   permit persons to whom the Software is furnished to do so, subject to
 *   the following conditions:
 *
 *   The above copyright notice and this permission notice shall be
 *   included in all copies or substantial portions of the Software.
 *
 *   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 *   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 *   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 *   IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 *   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 *   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 *   OTHER DEALINGS IN THE SOFTWARE.
 ***************************************************************************/

#include "CEGUI/NamedElement.h"

#include <boost/test/unit_test.hpp>

BOOST_AUTO_TEST_SUITE(NamedElement)

BOOST_AUTO_TEST_CASE(Children)
{
    CEGUI::NamedElement* root = new CEGUI::NamedElement("root");
    BOOST_CHECK_EQUAL(root->getName(), "root");
    BOOST_CHECK_EQUAL(root->getProperty("Name"), "root");
    CEGUI::NamedElement* child1 = new CEGUI::NamedElement("child");
    CEGUI::NamedElement* child2 = new CEGUI::NamedElement("child");
    root->addChild(child1);
    BOOST_CHECK_THROW(root->addChild(child2), CEGUI::AlreadyExistsException);
    root->removeChild("child");
    BOOST_CHECK_THROW(root->removeChild("child"), CEGUI::UnknownObjectException);
    root->addChild(child2);
    BOOST_CHECK_THROW(root->addChild(child1), CEGUI::AlreadyExistsException);

    delete child2;
    delete child1;
    delete root;
}

BOOST_AUTO_TEST_CASE(NamePath)
{
    CEGUI::NamedElement* root = new CEGUI::NamedElement("root");
    CEGUI::NamedElement* child = new CEGUI::NamedElement("child");
    CEGUI::NamedElement* inner_child = new CEGUI::NamedElement("inner_child");
    root->addChild(child);
    child->addChild(inner_child);

    BOOST_CHECK_EQUAL(child->getNamePath(), "root/child");
    BOOST_CHECK_EQUAL(child->getProperty("NamePath"), "root/child");
    BOOST_CHECK_EQUAL(inner_child->getNamePath(), "root/child/inner_child");

    BOOST_CHECK_EQUAL(root->getChildElement("child"), child);
    BOOST_CHECK_EQUAL(root->getChildElement("child/inner_child"), inner_child);
    BOOST_CHECK_EQUAL(child->getChildElement("inner_child"), inner_child);
    BOOST_CHECK_THROW(child->getChildElement("nonexistant"), CEGUI::UnknownObjectException);

    delete inner_child;
    delete child;
    delete root;
}

BOOST_AUTO_TEST_SUITE_END()
<|endoftext|>