<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/operators/beam_search_op.h"

#include <algorithm>
#include <map>
#include <string>
#include <vector>
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/op_registry.h"

#include <iostream>

namespace paddle {
namespace operators {

void BeamSearch::operator()(const framework::LoDTensor &pre_ids,
                            framework::LoDTensor *selected_ids,
                            framework::LoDTensor *selected_scores) {
  auto abs_lod = framework::ToAbsOffset(ids_->lod());
  auto &high_level = abs_lod[lod_level_];

  auto items = SelectTopBeamSizeItems();
  auto selected_items = ToMap(items, high_level.back());
  VLOG(3) << "selected_items:";
  for (size_t i = 0; i < selected_items.size(); ++i) {
    VLOG(3) << "offset:" << i;
    for (auto &item : selected_items[i]) {
      VLOG(3) << ItemToString(item);
    }
  }
  PruneEndidCandidates(pre_ids, &selected_items);
  // calculate the output tensor's height
  size_t num_instances = std::accumulate(
      std::begin(selected_items), std::end(selected_items), 0,
      [](size_t a, std::vector<Item> &b) { return a + b.size(); });
  // the output tensor shape should be [num_instances, 1]
  auto dims = framework::make_ddim(
      std::vector<int64_t>({static_cast<int>(num_instances), 1}));
  selected_ids->Resize(dims);
  selected_scores->Resize(dims);

  std::map<size_t /*offset*/, std::vector<Item>> hash;
  framework::LoD new_lod;
  auto *ids_data = selected_ids->mutable_data<int64_t>(platform::CPUPlace());
  auto *scores_data =
      selected_scores->mutable_data<float>(platform::CPUPlace());

  // fill in data
  std::vector<size_t> low_level;
  size_t low_offset = 0;
  for (auto &items : selected_items) {
    low_level.push_back(low_offset);
    sort(items.begin(), items.end(), [](const Item &a, const Item &b) {
      if (a.offset < b.offset) {
        return true;
      }
      return a.id < b.id;
    });
    for (auto &item : items) {
      ids_data[low_offset] = item.id;
      scores_data[low_offset] = item.score;
      low_offset++;
    }
  }
  low_level.push_back(low_offset);

  // fill lod
  framework::LoD lod(2);
  lod[0].assign(high_level.begin(), high_level.end());
  lod[1].assign(low_level.begin(), low_level.end());
  if (!framework::CheckLoD(lod)) {
    PADDLE_THROW("lod %s is not right", framework::LoDToString(lod));
  }
  selected_ids->set_lod(lod);
  selected_scores->set_lod(lod);
}

int BeamSearch::PruneEndidCandidates(const framework::LoDTensor &pre_ids,
                                     std::vector<std::vector<Item>> *items) {
  auto *pre_ids_data = pre_ids.data<int64_t>();

  int res = 0;
  for (size_t offset = 0; offset < items->size(); offset++) {
    auto prefix_id = pre_ids_data[offset];
    if (prefix_id == end_id_) {
      items->at(offset).clear();
    } else {
      res++;
    }
  }

  return res;
}

std::vector<std::vector<BeamSearch::Item>> BeamSearch::ToMap(
    const std::vector<std::vector<Item>> &items, size_t element_num) {
  std::vector<std::vector<Item>> result;
  result.resize(element_num);
  for (auto &entries : items) {
    for (const auto &item : entries) {
      result[item.offset].push_back(item);
    }
  }
  return result;
}

std::vector<std::vector<BeamSearch::Item>>
BeamSearch::SelectTopBeamSizeItems() {
  std::vector<std::vector<Item>> result;
  std::vector<Item> items;
  // for each source sentence, select the top beam_size items across all
  // candidate sets.
  while (NextItemSet(&items)) {
    std::nth_element(std::begin(items), std::begin(items) + beam_size_,
                     std::end(items), [](const Item &a, const Item &b) {
                       // TODO(superjom) make score's comparation customizable.
                       // partial sort in descending order
                       return a.score > b.score;
                     });
    // prune the top beam_size items.
    if (items.size() > beam_size_) {
      items.resize(beam_size_);
    }
    result.emplace_back(items);
  }
  VLOG(3) << "SelectTopBeamSizeItems result size " << result.size();
  for (auto &items : result) {
    VLOG(3) << "item set:";
    for (auto &item : items) {
      VLOG(3) << ItemToString(item);
    }
  }

  return result;
}

// the candidates of a source
bool BeamSearch::NextItemSet(std::vector<BeamSearch::Item> *items) {
  if (sent_offset_ >= ids_->NumElements(lod_level_)) {
    return false;
  }
  // find the current candidates
  auto ids = *ids_;
  auto scores = *scores_;

  auto abs_lod = framework::ToAbsOffset(ids.lod());

  auto *ids_data = ids.data<int64_t>();
  auto *scores_data = scores.data<float>();

  size_t instance_dim = 1;
  for (int i = 1; i < ids.dims().size(); i++) {
    instance_dim *= ids.dims()[i];
  }

  items->clear();
  items->reserve(framework::product(ids.dims()));
  for (size_t offset = abs_lod[lod_level_][sent_offset_];
       offset < abs_lod[lod_level_][sent_offset_ + 1]; offset++) {
    for (size_t d = 0; d < instance_dim; d++) {
      const size_t dim_offset = offset * instance_dim + d;
      items->emplace_back(offset, ids_data[dim_offset],
                          scores_data[dim_offset]);
    }
  }

  sent_offset_++;
  return true;
}

std::ostream &operator<<(std::ostream &os, const BeamSearch::Item &item) {
  os << "{";
  os << "offset: " << item.offset << ", ";
  os << "id: " << item.id << ", ";
  os << "score: " << item.score << "";
  os << "}";

  return os;
}

std::string ItemToString(const BeamSearch::Item &item) {
  std::ostringstream stream;
  stream << item;
  return stream.str();
}

class BeamSearchOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  BeamSearchOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    // inputs and outputs stored in proto
    AddInput("pre_ids", "ids in previous step");
    AddInput("ids", "a LoDTensor of shape of [None,k]");
    AddInput("scores",
             "a LoDTensor that has the same shape and LoD with `ids`");
    AddOutput("selected_ids",
              "a LoDTensor that stores the IDs selected by beam search");
    AddOutput(
        "selected_scores",
        "a LoDTensor that has the same shape and LoD with `selected_ids`");

    // Attributes stored in AttributeMap
    AddAttr<int>("level", "the level of LoDTensor");
    AddAttr<int>("beam_size", "beam size for beam search");
    AddAttr<int>("end_id",
                 "the token id which indicates the end of a sequence");

    AddComment(
        "This is a beam search operator that help to generate sequences.");
  }
};

class BeamSearchOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

 protected:
  void InferShape(framework::InferShapeContext *ctx) const override {
    for (const std::string &arg :
         std::vector<std::string>({"pre_ids", "ids", "scores"})) {
      PADDLE_ENFORCE(ctx->HasInput(arg), "BeamSearch need input argument '%s'",
                     arg);
    }
    for (const std::string &arg :
         std::vector<std::string>({"selected_ids", "selected_scores"})) {
      PADDLE_ENFORCE(ctx->HasOutput(arg),
                     "BeamSearch need output argument '%s'", arg);
    }
    std::cout << "Done Infer Shape\n";
  }

  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext &ctx) const override {
    std::cout << "Get Expected type 1\n";
    framework::OpKernelType kt = framework::OpKernelType(
        framework::ToDataType(
            ctx.Input<framework::LoDTensor>("pre_ids")->type()),
        platform::CPUPlace());
    std::cout << "Get Expected type 2\n";
    return kt;
  }
};

class BeamSearchInferVarType : public framework::VarTypeInference {
 public:
  void operator()(const framework::OpDesc &op_desc,
                  framework::BlockDesc *block) const override {
    for (auto &o : op_desc.Output("selected_ids")) {
      block->Var(o)->SetType(framework::proto::VarType::LOD_TENSOR);
    }
    for (auto &o : op_desc.Output("selected_scores")) {
      block->Var(o)->SetType(framework::proto::VarType::LOD_TENSOR);
    }
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

REGISTER_OPERATOR(beam_search, ops::BeamSearchOp, ops::BeamSearchOpMaker,
                  ops::BeamSearchInferVarType);
REGISTER_OP_CPU_KERNEL(
    beam_search,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, float>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, double>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, int>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, int64_t>);
<commit_msg>Style fix<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/operators/beam_search_op.h"

#include <algorithm>
#include <map>
#include <string>
#include <vector>
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/op_registry.h"

#include <iostream>

namespace paddle {
namespace operators {

void BeamSearch::operator()(const framework::LoDTensor &pre_ids,
                            framework::LoDTensor *selected_ids,
                            framework::LoDTensor *selected_scores) {
  auto abs_lod = framework::ToAbsOffset(ids_->lod());
  auto &high_level = abs_lod[lod_level_];

  auto items = SelectTopBeamSizeItems();
  auto selected_items = ToMap(items, high_level.back());
  VLOG(3) << "selected_items:";
  for (size_t i = 0; i < selected_items.size(); ++i) {
    VLOG(3) << "offset:" << i;
    for (auto &item : selected_items[i]) {
      VLOG(3) << ItemToString(item);
    }
  }
  PruneEndidCandidates(pre_ids, &selected_items);
  // calculate the output tensor's height
  size_t num_instances = std::accumulate(
      std::begin(selected_items), std::end(selected_items), 0,
      [](size_t a, std::vector<Item> &b) { return a + b.size(); });
  // the output tensor shape should be [num_instances, 1]
  auto dims = framework::make_ddim(
      std::vector<int64_t>({static_cast<int>(num_instances), 1}));
  selected_ids->Resize(dims);
  selected_scores->Resize(dims);

  std::map<size_t /*offset*/, std::vector<Item>> hash;
  framework::LoD new_lod;
  auto *ids_data = selected_ids->mutable_data<int64_t>(platform::CPUPlace());
  auto *scores_data =
      selected_scores->mutable_data<float>(platform::CPUPlace());

  // fill in data
  std::vector<size_t> low_level;
  size_t low_offset = 0;
  for (auto &items : selected_items) {
    low_level.push_back(low_offset);
    sort(items.begin(), items.end(), [](const Item &a, const Item &b) {
      if (a.offset < b.offset) {
        return true;
      }
      return a.id < b.id;
    });
    for (auto &item : items) {
      ids_data[low_offset] = item.id;
      scores_data[low_offset] = item.score;
      low_offset++;
    }
  }
  low_level.push_back(low_offset);

  // fill lod
  framework::LoD lod(2);
  lod[0].assign(high_level.begin(), high_level.end());
  lod[1].assign(low_level.begin(), low_level.end());
  if (!framework::CheckLoD(lod)) {
    PADDLE_THROW("lod %s is not right", framework::LoDToString(lod));
  }
  selected_ids->set_lod(lod);
  selected_scores->set_lod(lod);
}

int BeamSearch::PruneEndidCandidates(const framework::LoDTensor &pre_ids,
                                     std::vector<std::vector<Item>> *items) {
  auto *pre_ids_data = pre_ids.data<int64_t>();

  int res = 0;
  for (size_t offset = 0; offset < items->size(); offset++) {
    auto prefix_id = pre_ids_data[offset];
    if (prefix_id == end_id_) {
      items->at(offset).clear();
    } else {
      res++;
    }
  }

  return res;
}

std::vector<std::vector<BeamSearch::Item>> BeamSearch::ToMap(
    const std::vector<std::vector<Item>> &items, size_t element_num) {
  std::vector<std::vector<Item>> result;
  result.resize(element_num);
  for (auto &entries : items) {
    for (const auto &item : entries) {
      result[item.offset].push_back(item);
    }
  }
  return result;
}

std::vector<std::vector<BeamSearch::Item>>
BeamSearch::SelectTopBeamSizeItems() {
  std::vector<std::vector<Item>> result;
  std::vector<Item> items;
  // for each source sentence, select the top beam_size items across all
  // candidate sets.
  while (NextItemSet(&items)) {
    std::nth_element(std::begin(items), std::begin(items) + beam_size_,
                     std::end(items), [](const Item &a, const Item &b) {
                       // TODO(superjom) make score's comparation customizable.
                       // partial sort in descending order
                       return a.score > b.score;
                     });
    // prune the top beam_size items.
    if (items.size() > beam_size_) {
      items.resize(beam_size_);
    }
    result.emplace_back(items);
  }
  VLOG(3) << "SelectTopBeamSizeItems result size " << result.size();
  for (auto &items : result) {
    VLOG(3) << "item set:";
    for (auto &item : items) {
      VLOG(3) << ItemToString(item);
    }
  }

  return result;
}

// the candidates of a source
bool BeamSearch::NextItemSet(std::vector<BeamSearch::Item> *items) {
  if (sent_offset_ >= ids_->NumElements(lod_level_)) {
    return false;
  }
  // find the current candidates
  auto ids = *ids_;
  auto scores = *scores_;

  auto abs_lod = framework::ToAbsOffset(ids.lod());

  auto *ids_data = ids.data<int64_t>();
  auto *scores_data = scores.data<float>();

  size_t instance_dim = 1;
  for (int i = 1; i < ids.dims().size(); i++) {
    instance_dim *= ids.dims()[i];
  }

  items->clear();
  items->reserve(framework::product(ids.dims()));
  for (size_t offset = abs_lod[lod_level_][sent_offset_];
       offset < abs_lod[lod_level_][sent_offset_ + 1]; offset++) {
    for (size_t d = 0; d < instance_dim; d++) {
      const size_t dim_offset = offset * instance_dim + d;
      items->emplace_back(offset, ids_data[dim_offset],
                          scores_data[dim_offset]);
    }
  }

  sent_offset_++;
  return true;
}

std::ostream &operator<<(std::ostream &os, const BeamSearch::Item &item) {
  os << "{";
  os << "offset: " << item.offset << ", ";
  os << "id: " << item.id << ", ";
  os << "score: " << item.score << "";
  os << "}";

  return os;
}

std::string ItemToString(const BeamSearch::Item &item) {
  std::ostringstream stream;
  stream << item;
  return stream.str();
}

class BeamSearchOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  BeamSearchOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    // inputs and outputs stored in proto
    AddInput("pre_ids", "ids in previous step");
    AddInput("ids", "a LoDTensor of shape of [None,k]");
    AddInput("scores",
             "a LoDTensor that has the same shape and LoD with `ids`");
    AddOutput("selected_ids",
              "a LoDTensor that stores the IDs selected by beam search");
    AddOutput(
        "selected_scores",
        "a LoDTensor that has the same shape and LoD with `selected_ids`");

    // Attributes stored in AttributeMap
    AddAttr<int>("level", "the level of LoDTensor");
    AddAttr<int>("beam_size", "beam size for beam search");
    AddAttr<int>("end_id",
                 "the token id which indicates the end of a sequence");

    AddComment(
        "This is a beam search operator that help to generate sequences.");
  }
};

class BeamSearchOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

 protected:
  void InferShape(framework::InferShapeContext *ctx) const override {
    for (const std::string &arg :
         std::vector<std::string>({"pre_ids", "ids", "scores"})) {
      PADDLE_ENFORCE(ctx->HasInput(arg), "BeamSearch need input argument '%s'",
                     arg);
    }
    for (const std::string &arg :
         std::vector<std::string>({"selected_ids", "selected_scores"})) {
      PADDLE_ENFORCE(ctx->HasOutput(arg),
                     "BeamSearch need output argument '%s'", arg);
    }
    std::cout << "Done Infer Shape\n";
  }

  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext &ctx) const override {
    std::cout << "Get Expected type 1\n";
    framework::OpKernelType kt = framework::OpKernelType(
        framework::ToDataType(
            ctx.Input<framework::LoDTensor>("pre_ids")->type()),
        platform::CPUPlace());
    std::cout << "Get Expected type 2\n";
    return kt;
  }
};

class BeamSearchInferVarType : public framework::VarTypeInference {
 public:
  void operator()(const framework::OpDesc &op_desc,
                  framework::BlockDesc *block) const override {
    for (auto &o : op_desc.Output("selected_ids")) {
      block->Var(o)->SetType(framework::proto::VarType::LOD_TENSOR);
    }
    for (auto &o : op_desc.Output("selected_scores")) {
      block->Var(o)->SetType(framework::proto::VarType::LOD_TENSOR);
    }
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

REGISTER_OPERATOR(beam_search, ops::BeamSearchOp, ops::BeamSearchOpMaker,
                  ops::BeamSearchInferVarType);
REGISTER_OP_CPU_KERNEL(
    beam_search,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, float>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, double>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, int>,
    ops::BeamSearchOpKernel<paddle::platform::CPUDeviceContext, int64_t>);
<|endoftext|>


<commit_before><commit_msg>test something else to fix dist crashes<commit_after><|endoftext|>


<commit_before>#ifndef VIGRA_MULTI_ARRAY_MAP_HXX
#define VIGRA_MULTI_ARRAY_MAP_HXX

/* boost */
#include <boost/function.hpp>

/* vigra */
#include <vigra/multi_array.hxx>
#include <vigra/accumulator.hxx>

/* vigra - merge graph */
#include <vigra/merge_graph/min_indexed_pq.hxx>

namespace vigra{
namespace view_maps {

    template<unsigned int DIM,class T>
    class SumMap;

    template<class T>
    class SumMap<0,T> : public MultiArrayView<1,T> {
    private:
        SumMap();                               // non empty-construction
        SumMap( const SumMap& other );          // non construction-copyable
        SumMap& operator=( const SumMap& );     // non copyable
    public:
        typedef MultiArrayView<1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        //const value_type & operator[](const IdType label)const{
        //  return this->operator(label);
        //}

        void merge(const IdType a,const IdType b){
            this->operator()(a)+=this->operator()(b);
        }

        template<class CB>
        CB mergeCallback(){
            CB cb;
            cb = boost::bind(boost::mem_fn(&SumMap<0,T>::merge), this , _1,_2);
            return cb;
        }

        SumMap(ArrayViewType & array)
        :   ArrayViewType(array){
        }
    };

    template<unsigned int DIM, class T>
    class SumMap : public MultiArrayView<DIM+1,T> {
    private:
        SumMap();                               // non empty-construction
        SumMap( const SumMap& other );          // non construction-copyable
        SumMap& operator=( const SumMap& );     // non copyable
    public:
        typedef MultiArrayView<DIM+1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        //const value_type & operator[](const IdType label)const{
        //  return this->operator(label);
        //}

        void merge(const IdType a,const IdType b){
            this->bindInner(a)+=this->bindInner(b);
        }

        template<class CB>
        CB mergeCallback(){
            CB cb;
            cb = boost::bind(boost::mem_fn(&SumMap<DIM,T>::merge), this , _1,_2);
            return cb;
        }

        SumMap(ArrayViewType & array)
        :   ArrayViewType(array){
        }
    };

    template<unsigned int DIM,class T,class WEIGHT_MAP>
    class WeightedMeanMap;

    template<class T,class WEIGHT_MAP>
    class WeightedMeanMap<0,T,WEIGHT_MAP> : public MultiArrayView<1,T> {
    private:
        WeightedMeanMap();                                      // non empty-construction
        WeightedMeanMap( const WeightedMeanMap& other );        // non construction-copyable
        WeightedMeanMap& operator=( const WeightedMeanMap& );   // non copyable
    public:
        typedef WEIGHT_MAP WeightMapType;
        typedef MultiArrayView<1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        void merge(const IdType a,const IdType b){
            const T va=this->operator()(a);
            const T vb=this->operator()(b);
            const T wa=weightMap_(a);
            const T wb=weightMap_(b);
            this->operator()(a)=(va*wa + vb*wb)/(wa+wb);
        }

        template<class CB>
        CB mergeCallback(){
            return  boost::bind(boost::mem_fn(&WeightedMeanMap<0,T,WEIGHT_MAP>::merge), this , _1,_2);
        }

        WeightedMeanMap(ArrayViewType & array,const WeightMapType & weightMap)
        :   ArrayViewType(array),
            weightMap_(weightMap){
        }
    private:
        const WeightMapType & weightMap_;
    };

    template<unsigned int DIM,class T,class WEIGHT_MAP>
    class WeightedMeanMap : public MultiArrayView<DIM+1,T> {
    private:
        WeightedMeanMap();                                      // non empty-construction
        WeightedMeanMap( const WeightedMeanMap& other );        // non construction-copyable
        WeightedMeanMap& operator=( const WeightedMeanMap& );   // non copyable
    public:
        typedef WEIGHT_MAP WeightMapType;
        typedef MultiArrayView<DIM+1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        void merge(const IdType a,const IdType b){
            const T wa=weightMap_(a);
            const T wb=weightMap_(b);
            vigra::MultiArrayView<DIM,T> va = this->bindInner(a);
            vigra::MultiArray<DIM,T> vb = this->bindInner(b);
            va=(va*wa+vb*wb);
            //va*=wa;
            //vb*=wb;
            //va+=vb;
            //va/=(wa+wb);
            //vb/=wb;
        }

        template<class CB>
        CB mergeCallback(){
            return  boost::bind(boost::mem_fn(&WeightedMeanMap<DIM,T,WEIGHT_MAP>::merge), this , _1,_2);
        }

        WeightedMeanMap(ArrayViewType & array,const WeightMapType & weightMap)
        :   ArrayViewType(array),
            weightMap_(weightMap){
        }
    private:
        const WeightMapType & weightMap_;
    };

    template<class T,class MERGE_GRAPH>
    class MinWeightEdgeMapSimple : public MultiArrayView<1,T> {
    private:
        MinWeightEdgeMapSimple();                                           // non empty-construction
        MinWeightEdgeMapSimple( const MinWeightEdgeMapSimple& other );          // non construction-copyable
        MinWeightEdgeMapSimple& operator=( const MinWeightEdgeMapSimple& );     // non copyable
    public:
        typedef MERGE_GRAPH MergeGraphType;
        typedef typename MergeGraphType::Edge   Edge;
        typedef typename MergeGraphType::EdgeIt EdgeIt;
        typedef MultiArrayView<1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        void operator[](const Edge edge)const{
            return MultiArrayView<1,T>::operator()(mergeGraph_.id(edge));
        }

        void mergeEdges(const IdType a,const IdType b){
            pq_.deleteValue(b);
            changePqWeight(a,this->operator()(a));
        }
        void eraseEdge(const IdType label){
            pq_.deleteValue(label);
        }

        template<class CB>
        CB eraseEdgeCallback(){
            return  boost::bind(boost::mem_fn(&MinWeightEdgeMapSimple<T,MERGE_GRAPH>::eraseEdge), this , _1);
        }

        template<class CB>
        CB mergeEdgeCallback(){
            return  boost::bind(boost::mem_fn(&MinWeightEdgeMapSimple<T,MERGE_GRAPH>::mergeEdges), this , _1,_2);
        }

        MinWeightEdgeMapSimple(const MergeGraphType & mergeGraph,ArrayViewType & array)
        :   ArrayViewType(array),
            mergeGraph_(mergeGraph),
            pq_(mergeGraph.maxEdgeId()+1){
            for(EdgeIt e(mergeGraph);e!=lemon::INVALID;++e){
                const Edge edge = *e;
                const IdType edgeId = mergeGraph_.id(edge);
                pq_.insert(edgeId,this->operator()(edgeId));
            }
        }

        IdType minWeightEdgeLabel(){
            IdType minLabel = pq_.minIndex();
            while(mergeGraph_.hasEdgeId(minLabel)==false){
                pq_.deleteValue(minLabel);
                IdType minLabel = pq_.minIndex();
            }
            return minLabel;
        }

    private:
        void changePqWeight(const IdType l,const T newWeigt){
            pq_.changeValue(l,newWeigt);
        }
        const MergeGraphType & mergeGraph_;
        vigra::MinIndexedPQ<T>     pq_;
    };

    template<class T,class MERGE_GRAPH,class EDGE_MAP,class NODE_MAP>
    class MinWeightEdgeMap : public MultiArrayView<1,T> {
    private:
        MinWeightEdgeMap();                                         // non empty-construction
        MinWeightEdgeMap( const MinWeightEdgeMap& other );          // non construction-copyable
        MinWeightEdgeMap& operator=( const MinWeightEdgeMap& );     // non copyable
    public:
        typedef MERGE_GRAPH MergeGraphType;
        typedef typename MergeGraphType::Edge Edge;
        typedef EDGE_MAP    EdgeMapType;
        typedef NODE_MAP    NodeMapType;
        typedef MultiArrayView<1,T> ArrayViewType;
        typedef Int32 IdType;
        typedef T value_type;

        void operator[](const Edge edge)const{
            return MultiArrayView<1,T>::operator()(mergeGraph_.id(edge));
        }

        void mergeEdges(const IdType a,const IdType b){
            this->operator()(a)=getMixedWeight(a);
            pq_.deleteValue(b);
        }
        void eraseEdge(const IdType label){
            pq_.deleteValue(label);
        }

        template<class CB>
        CB eraseEdgeCallback(){
            return  boost::bind(boost::mem_fn(&MinWeightEdgeMap<T,MERGE_GRAPH,EDGE_MAP,NODE_MAP>::eraseEdge), this , _1);
        }

        template<class CB>
        CB mergeEdgeCallback(){
            return  boost::bind(boost::mem_fn(&MinWeightEdgeMap<T,MERGE_GRAPH,EDGE_MAP,NODE_MAP>::mergeEdges), this , _1,_2);
        }

        MinWeightEdgeMap(const MergeGraphType & mergeGraph,ArrayViewType & array,const EdgeMapType & edgeMap,const NodeMapType & nodeMap)
        :   ArrayViewType(array),
            mergeGraph_(mergeGraph),
            edgeMap_(edgeMap),
            nodeMap_(nodeMap),
            pq_(mergeGraph.initNumberOfEdges()){

            // initalize mixed weights  and initalize pq
            for(IdType l=0;l<mergeGraph_.initNumberOfEdges();++l){
                const T mixedWeight = getMixedWeight(l);
                this->operator()(l)=mixedWeight;
                pq_.insert(l,mixedWeight);
            }
        }

        IdType minWeightEdgeLabel(){
            IdType minLabel = pq_.minIndex();
            while(mergeGraph_.hasEdgeId(minLabel)==false){
                pq_.deleteValue(minLabel);
                IdType minLabel = pq_.minIndex();
            }
            return minLabel;
        }
    private:
        T getMixedWeight(const IdType label)const{
            return edgeMap_(label);
        }

        void changePqWeight(const IdType l,const T newWeigt){
            pq_.changeValue(l,newWeigt);
        }

        const MergeGraphType & mergeGraph_;
        const EdgeMapType    & edgeMap_;
        const NodeMapType    & nodeMap_;
        vigra::MinIndexedPQ<T>     pq_;
    };

} // end namespace view_maps
} // end namespace vigra

#endif // VIGRA_MULTI_ARRAY_MAP_HXX<commit_msg>deleted unused file<commit_after><|endoftext|>


<commit_before>#include "ILatentPrior.h"
#include <Utils/counters.h>

namespace smurff {

ILatentPrior::ILatentPrior(TrainSession &trainSession, uint32_t mode, std::string name)
   : m_session(trainSession), m_mode(mode), m_name(name)
{

}

void ILatentPrior::init()
{
   rrs.init(Vector::Zero(num_latent()));
   MMs.init(Matrix::Zero(num_latent(), num_latent()));

   //this is some new initialization
   init_Usum();
}

const Config &ILatentPrior::getConfig() const
{
   return m_session.getConfig();
}

const Model& ILatentPrior::model() const
{
   return m_session.model();
}

Model& ILatentPrior::model()
{
   return m_session.model();
}

double ILatentPrior::predict(const PVec<> &pos) const
{
    return model().predict(pos);
}

Data& ILatentPrior::data() const
{
   return m_session.data();
}

INoiseModel& ILatentPrior::noise()
{
   return data().noise();
}

Matrix &ILatentPrior::U()
{
   return model().U(m_mode);
}

const Matrix &ILatentPrior::U() const
{
   return model().U(m_mode);
}

//return V matrices in the model opposite to mode
VMatrixIterator<Matrix> ILatentPrior::Vbegin()
{
   return model().Vbegin(m_mode);
}

VMatrixIterator<Matrix> ILatentPrior::Vend()
{
   return model().Vend();
}

ConstVMatrixIterator<Matrix> ILatentPrior::CVbegin() const
{
   return model().CVbegin(m_mode);
}

ConstVMatrixIterator<Matrix> ILatentPrior::CVend() const
{
   return model().CVend();
}

int ILatentPrior::num_latent() const
{
   return model().nlatent();
}

int ILatentPrior::num_item() const
{
   return model().U(m_mode).rows();
}

std::ostream &ILatentPrior::info(std::ostream &os, std::string indent)
{
   os << indent << m_mode << ": " << m_name << std::endl;
   return os;
}

bool ILatentPrior::run_slave()
{
   return false;
}

void ILatentPrior::sample_latents()
{
   COUNTER("sample_latents");
   data().update_pnm(model(), m_mode);

   // for effiency, we keep + update Urow and UUrow by every thread
   thread_vector<Vector> Urow(Vector::Zero(num_latent()));
   thread_vector<Matrix> UUrow(Matrix::Zero(num_latent(), num_latent()));

   #pragma omp parallel for schedule(guided)
   for(int n = 0; n < U().rows(); n++)
   #pragma omp task
   {
      COUNTER("sample_latent");
      sample_latent(n);
      const auto &row = U().row(n);
      Urow.local().noalias() += row;
      UUrow.local().noalias() += row.transpose() * row;

      if (m_session.inSamplingPhase())
         model().updateAggr(m_mode, n);
   }

   if (m_session.inSamplingPhase())
      model().updateAggr(m_mode);

   Usum  = Urow.combine();
   UUsum = UUrow.combine();
}

bool ILatentPrior::save(SaveState &sf) const
{
    return false;
}

void ILatentPrior::restore(const SaveState &sf)
{
    init_Usum();
    update_prior();
}

void ILatentPrior::init_Usum()
{
    Usum = U().rowwise().sum();
    UUsum = U().transpose() * U();
}
} // end namespace smurff
<commit_msg>WIP: FIX Usum to be colwise<commit_after>#include "ILatentPrior.h"
#include <Utils/counters.h>

namespace smurff {

ILatentPrior::ILatentPrior(TrainSession &trainSession, uint32_t mode, std::string name)
   : m_session(trainSession), m_mode(mode), m_name(name)
{

}

void ILatentPrior::init()
{
   rrs.init(Vector::Zero(num_latent()));
   MMs.init(Matrix::Zero(num_latent(), num_latent()));

   //this is some new initialization
   init_Usum();
}

const Config &ILatentPrior::getConfig() const
{
   return m_session.getConfig();
}

const Model& ILatentPrior::model() const
{
   return m_session.model();
}

Model& ILatentPrior::model()
{
   return m_session.model();
}

double ILatentPrior::predict(const PVec<> &pos) const
{
    return model().predict(pos);
}

Data& ILatentPrior::data() const
{
   return m_session.data();
}

INoiseModel& ILatentPrior::noise()
{
   return data().noise();
}

Matrix &ILatentPrior::U()
{
   return model().U(m_mode);
}

const Matrix &ILatentPrior::U() const
{
   return model().U(m_mode);
}

//return V matrices in the model opposite to mode
VMatrixIterator<Matrix> ILatentPrior::Vbegin()
{
   return model().Vbegin(m_mode);
}

VMatrixIterator<Matrix> ILatentPrior::Vend()
{
   return model().Vend();
}

ConstVMatrixIterator<Matrix> ILatentPrior::CVbegin() const
{
   return model().CVbegin(m_mode);
}

ConstVMatrixIterator<Matrix> ILatentPrior::CVend() const
{
   return model().CVend();
}

int ILatentPrior::num_latent() const
{
   return model().nlatent();
}

int ILatentPrior::num_item() const
{
   return model().U(m_mode).rows();
}

std::ostream &ILatentPrior::info(std::ostream &os, std::string indent)
{
   os << indent << m_mode << ": " << m_name << std::endl;
   return os;
}

bool ILatentPrior::run_slave()
{
   return false;
}

void ILatentPrior::sample_latents()
{
   COUNTER("sample_latents");
   data().update_pnm(model(), m_mode);

   // for effiency, we keep + update Urow and UUrow by every thread
   thread_vector<Vector> Urow(Vector::Zero(num_latent()));
   thread_vector<Matrix> UUrow(Matrix::Zero(num_latent(), num_latent()));

   #pragma omp parallel for schedule(guided)
   for(int n = 0; n < U().rows(); n++)
   #pragma omp task
   {
      COUNTER("sample_latent");
      sample_latent(n);
      const auto &row = U().row(n);
      Urow.local().noalias() += row;
      UUrow.local().noalias() += row.transpose() * row;

      if (m_session.inSamplingPhase())
         model().updateAggr(m_mode, n);
   }

   if (m_session.inSamplingPhase())
      model().updateAggr(m_mode);

   Usum  = Urow.combine();
   UUsum = UUrow.combine();
}

bool ILatentPrior::save(SaveState &sf) const
{
    return false;
}

void ILatentPrior::restore(const SaveState &sf)
{
    init_Usum();
    update_prior();
}

void ILatentPrior::init_Usum()
{
    Usum = U().colwise().sum();
    UUsum = U().transpose() * U();
}
} // end namespace smurff
<|endoftext|>


<commit_before>#include "paddle/framework/grad_op_builder.h"
#include <gtest/gtest.h>
#include "paddle/framework/op_registry.h"
#include "paddle/framework/operator.h"

USE_OP(add_two);

namespace paddle {
namespace framework {

class NOP : public OperatorBase {
 public:
  void InferShape(const Scope &scope) const override {}
  void Run(const Scope &scope,
           const platform::DeviceContext &dev_ctx) const override {}
};

class MutiInOutOpMaker : public OpProtoAndCheckerMaker {
 public:
  MutiInOutOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("In1", "a single input");
    AddInput("In2_mult", "a multiple input").SetMultiple();
    AddInput("In3", "another single input");
    AddOutput("Out1", "a single output");
    AddOutput("Out2_mult", "a multiple output").SetMultiple();
    AddComment("test op with multiple inputs and outputs");
  }
};

class IOIgnoredOpMaker : public OpProtoAndCheckerMaker {
 public:
  IOIgnoredOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("In1", "a single input");
    AddInput("In2_mult", "a multiple input").SetMultiple().IgnoreGradient();
    AddInput("In3_mult", "another multiple input").SetMultiple();
    AddOutput("Out1_mult", "a multiple output").SetMultiple();
    AddOutput("Out2", "a single output").IgnoreGradient();
    AddComment("op with inputs and outputs ignored in gradient calculating");
  }
};

}  // namespace framework
}  // namespace paddle

namespace f = paddle::framework;

TEST(GradOpBuilder, AddTwo) {
  std::shared_ptr<f::OperatorBase> add_op(
      f::OpRegistry::CreateOp("add_two", {"x", "y"}, {"out"}, {}));
  std::shared_ptr<f::OperatorBase> grad_add_op =
      f::OpRegistry::CreateGradOp(*add_op);
  EXPECT_EQ(static_cast<int>(grad_add_op->inputs_.size()), 4);
  EXPECT_EQ(static_cast<int>(grad_add_op->outputs_.size()), 2);
  EXPECT_EQ(grad_add_op->Input("X"), "x");
  EXPECT_EQ(grad_add_op->Input("Y"), "y");
  EXPECT_EQ(grad_add_op->Input("Out"), "out");
  EXPECT_EQ(grad_add_op->Input("Out@GRAD"), "out@GRAD");
  EXPECT_EQ(grad_add_op->Output("X@GRAD"), "x@GRAD");
  EXPECT_EQ(grad_add_op->Output("Y@GRAD"), "y@GRAD");
}

REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker);
REGISTER_GRADIENT_OP(mult_io, mult_io_grad, f::NOP);
REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker);
REGISTER_GRADIENT_OP(io_ignored, io_ignored_grad, f::NOP);

TEST(GradOpBuilder, MutiInOut) {
  f::AttributeMap attrs{{"input_format", std::vector<int>{0, 1, 4, 5}},
                        {"output_format", std::vector<int>{0, 1, 3}}};
  std::shared_ptr<f::OperatorBase> test_op(f::OpRegistry::CreateOp(
      "mult_io", {"in1", "in2_1", "in2_2", "in2_3", "in3"},
      {"out1", "out2_1", "out2_2"}, attrs));
  std::shared_ptr<f::OperatorBase> grad_test_op =
      f::OpRegistry::CreateGradOp(*test_op);

  ASSERT_EQ(grad_test_op->inputs_.size(), 5UL + 3UL + 3UL);
  EXPECT_EQ(grad_test_op->Input("In1"), "in1");
  EXPECT_EQ(grad_test_op->Inputs("In2_mult"),
            std::vector<std::string>({"in2_1", "in2_2", "in2_3"}));
  EXPECT_EQ(grad_test_op->Input("In3"), "in3");
  EXPECT_EQ(grad_test_op->Input("Out1"), "out1");
  EXPECT_EQ(grad_test_op->Inputs("Out2_mult"),
            std::vector<std::string>({"out2_1", "out2_2"}));
  EXPECT_EQ(grad_test_op->Input("Out1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
            "out1" + f::OperatorBase::GRAD_VAR_SUFFIX());
  EXPECT_EQ(
      grad_test_op->Inputs("Out2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
      std::vector<std::string>(
          {"out2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
           "out2_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));

  ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
  EXPECT_EQ(grad_test_op->Output("In1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
            "in1" + f::OperatorBase::GRAD_VAR_SUFFIX());
  EXPECT_EQ(
      grad_test_op->Outputs("In2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
      std::vector<std::string>({"in2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
                                "in2_2" + f::OperatorBase::GRAD_VAR_SUFFIX(),
                                "in2_3" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
  EXPECT_EQ(grad_test_op->Output("In3" + f::OperatorBase::GRAD_VAR_SUFFIX()),
            "in3" + f::OperatorBase::GRAD_VAR_SUFFIX());
}

TEST(GradOpBuilder, IOIgnoredInGradient) {
  f::AttributeMap attrs{{"input_format", std::vector<int>{0, 1, 3, 5}},
                        {"output_format", std::vector<int>{0, 2, 3}}};
  std::shared_ptr<f::OperatorBase> test_op(f::OpRegistry::CreateOp(
      "io_ignored", {"in1", "in2_1", "in2_2", "in3_1", "in3_2"},
      {"out1_1", "out1_2", "out2"}, attrs));
  std::shared_ptr<f::OperatorBase> grad_test_op =
      f::OpRegistry::CreateGradOp(*test_op);

  // 'In2' and 'Out2' are ignored in gradient calculating
  ASSERT_EQ(grad_test_op->inputs_.size(), 5UL + 3UL + 3UL);
  EXPECT_EQ(grad_test_op->Input("In1"), "in1");
  EXPECT_EQ(grad_test_op->Inputs("In2_mult"),
            std::vector<std::string>({f::OperatorBase::EMPTY_VAR_NAME(),
                                      f::OperatorBase::EMPTY_VAR_NAME()}));
  EXPECT_EQ(grad_test_op->Inputs("In3_mult"),
            std::vector<std::string>({"in3_1", "in3_2"}));
  EXPECT_EQ(grad_test_op->Inputs("Out1_mult"),
            std::vector<std::string>({"out1_1", "out1_2"}));
  EXPECT_EQ(grad_test_op->Input("Out2"), f::OperatorBase::EMPTY_VAR_NAME());
  EXPECT_EQ(
      grad_test_op->Inputs("Out1_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
      std::vector<std::string>(
          {"out1_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
           "out1_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
  EXPECT_EQ(grad_test_op->Input("Out2" + f::OperatorBase::GRAD_VAR_SUFFIX()),
            "out2" + f::OperatorBase::GRAD_VAR_SUFFIX());

  ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
  EXPECT_EQ(grad_test_op->Output("In1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
            "in1" + f::OperatorBase::GRAD_VAR_SUFFIX());
  EXPECT_EQ(
      grad_test_op->Outputs("In2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
      std::vector<std::string>({"in2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
                                "in2_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
  EXPECT_EQ(
      grad_test_op->Outputs("In3_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
      std::vector<std::string>({"in3_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
                                "in3_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
}
<commit_msg>Update new source files<commit_after>#include "paddle/framework/grad_op_builder.h"
#include <gtest/gtest.h>
#include "paddle/framework/op_registry.h"
#include "paddle/framework/operator.h"

USE_OP(add_two);

namespace paddle {
namespace framework {

class NOP : public OperatorBase {
 public:
  void InferShape(const Scope &scope) const override {}
  void Run(const Scope &scope,
           const platform::DeviceContext &dev_ctx) const override {}
};

class MutiInOutOpMaker : public OpProtoAndCheckerMaker {
 public:
  MutiInOutOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("In1", "a single input");
    AddInput("In2_mult", "a multiple input").SetMultiple();
    AddInput("In3", "another single input");
    AddOutput("Out1", "a single output");
    AddOutput("Out2_mult", "a multiple output").SetMultiple();
    AddComment("test op with multiple inputs and outputs");
  }
};

class IOIgnoredOpMaker : public OpProtoAndCheckerMaker {
 public:
  IOIgnoredOpMaker(OpProto *proto, OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("In1", "a single input");
    AddInput("In2_mult", "a multiple input").SetMultiple().IgnoreGradient();
    AddInput("In3_mult", "another multiple input").SetMultiple();
    AddOutput("Out1_mult", "a multiple output").SetMultiple();
    AddOutput("Out2", "a single output").IgnoreGradient();
    AddComment("op with inputs and outputs ignored in gradient calculating");
  }
};

}  // namespace framework
}  // namespace paddle

namespace f = paddle::framework;

TEST(GradOpBuilder, AddTwo) {
  std::shared_ptr<f::OperatorBase> add_op(
      f::OpRegistry::CreateOp("add_two", {"x", "y"}, {"out"}, {}));
  std::shared_ptr<f::OperatorBase> grad_add_op =
      f::OpRegistry::CreateGradOp(*add_op);
  EXPECT_EQ(static_cast<int>(grad_add_op->inputs_.size()), 4);
  EXPECT_EQ(static_cast<int>(grad_add_op->outputs_.size()), 2);
  EXPECT_EQ(grad_add_op->Input("X"), "x");
  EXPECT_EQ(grad_add_op->Input("Y"), "y");
  EXPECT_EQ(grad_add_op->Input("Out"), "out");
  EXPECT_EQ(grad_add_op->Input("Out@GRAD"), "out@GRAD");
  EXPECT_EQ(grad_add_op->Output("X@GRAD"), "x@GRAD");
  EXPECT_EQ(grad_add_op->Output("Y@GRAD"), "y@GRAD");
}

REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker);
REGISTER_GRADIENT_OP(mult_io, mult_io_grad, f::NOP);
REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker);
REGISTER_GRADIENT_OP(io_ignored, io_ignored_grad, f::NOP);

TEST(GradOpBuilder, MutiInOut) {
  f::AttributeMap attrs{{"input_format", std::vector<int>{0, 1, 4, 5}},
                        {"output_format", std::vector<int>{0, 1, 3}}};
  std::shared_ptr<f::OperatorBase> test_op(f::OpRegistry::CreateOp(
      "mult_io", {"in1", "in2_1", "in2_2", "in2_3", "in3"},
      {"out1", "out2_1", "out2_2"}, attrs));
  std::shared_ptr<f::OperatorBase> grad_test_op =
      f::OpRegistry::CreateGradOp(*test_op);

  ASSERT_EQ(grad_test_op->inputs_.size(), 5UL + 3UL + 3UL);
  EXPECT_EQ(grad_test_op->Input("In1"), "in1");
  EXPECT_EQ(grad_test_op->Inputs("In2_mult"),
            std::vector<std::string>({"in2_1", "in2_2", "in2_3"}));
  EXPECT_EQ(grad_test_op->Input("In3"), "in3");
  EXPECT_EQ(grad_test_op->Input("Out1"), "out1");
  EXPECT_EQ(grad_test_op->Inputs("Out2_mult"),
            std::vector<std::string>({"out2_1", "out2_2"}));
  EXPECT_EQ(grad_test_op->Input("Out1" + f::kGradVarSuffix),
            "out1" + f::kGradVarSuffix);
  EXPECT_EQ(grad_test_op->Inputs("Out2_mult" + f::kGradVarSuffix),
            std::vector<std::string>(
                {"out2_1" + f::kGradVarSuffix, "out2_2" + f::kGradVarSuffix}));

  ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
  EXPECT_EQ(grad_test_op->Output("In1" + f::kGradVarSuffix),
            "in1" + f::kGradVarSuffix);
  EXPECT_EQ(grad_test_op->Outputs("In2_mult" + f::kGradVarSuffix),
            std::vector<std::string>({"in2_1" + f::kGradVarSuffix,
                                      "in2_2" + f::kGradVarSuffix,
                                      "in2_3" + f::kGradVarSuffix}));
  EXPECT_EQ(grad_test_op->Output("In3" + f::kGradVarSuffix),
            "in3" + f::kGradVarSuffix);
}

TEST(GradOpBuilder, IOIgnoredInGradient) {
  f::AttributeMap attrs{{"input_format", std::vector<int>{0, 1, 3, 5}},
                        {"output_format", std::vector<int>{0, 2, 3}}};
  std::shared_ptr<f::OperatorBase> test_op(f::OpRegistry::CreateOp(
      "io_ignored", {"in1", "in2_1", "in2_2", "in3_1", "in3_2"},
      {"out1_1", "out1_2", "out2"}, attrs));
  std::shared_ptr<f::OperatorBase> grad_test_op =
      f::OpRegistry::CreateGradOp(*test_op);

  // 'In2' and 'Out2' are ignored in gradient calculating
  ASSERT_EQ(grad_test_op->inputs_.size(), 5UL + 3UL + 3UL);
  EXPECT_EQ(grad_test_op->Input("In1"), "in1");
  EXPECT_EQ(grad_test_op->Inputs("In2_mult"),
            std::vector<std::string>({f::kEmptyVarName, f::kEmptyVarName}));
  EXPECT_EQ(grad_test_op->Inputs("In3_mult"),
            std::vector<std::string>({"in3_1", "in3_2"}));
  EXPECT_EQ(grad_test_op->Inputs("Out1_mult"),
            std::vector<std::string>({"out1_1", "out1_2"}));
  EXPECT_EQ(grad_test_op->Input("Out2"), f::kEmptyVarName);
  EXPECT_EQ(grad_test_op->Inputs("Out1_mult" + f::kGradVarSuffix),
            std::vector<std::string>(
                {"out1_1" + f::kGradVarSuffix, "out1_2" + f::kGradVarSuffix}));
  EXPECT_EQ(grad_test_op->Input("Out2" + f::kGradVarSuffix),
            "out2" + f::kGradVarSuffix);

  ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
  EXPECT_EQ(grad_test_op->Output("In1" + f::kGradVarSuffix),
            "in1" + f::kGradVarSuffix);
  EXPECT_EQ(grad_test_op->Outputs("In2_mult" + f::kGradVarSuffix),
            std::vector<std::string>(
                {"in2_1" + f::kGradVarSuffix, "in2_2" + f::kGradVarSuffix}));
  EXPECT_EQ(grad_test_op->Outputs("In3_mult" + f::kGradVarSuffix),
            std::vector<std::string>(
                {"in3_1" + f::kGradVarSuffix, "in3_2" + f::kGradVarSuffix}));
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/python/numpy_interop.h"

#include "arrow/python/numpy_convert.h"

#include <cstdint>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

#include "arrow/buffer.h"
#include "arrow/tensor.h"
#include "arrow/type.h"

#include "arrow/python/common.h"
#include "arrow/python/pyarrow.h"
#include "arrow/python/type_traits.h"

namespace arrow {
namespace py {

bool is_contiguous(PyObject* array) {
  if (PyArray_Check(array)) {
    return (PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
            (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0;
  } else {
    return false;
  }
}

NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
  arr_ = ao;
  Py_INCREF(ao);

  if (PyArray_Check(ao)) {
    PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);
    data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(ndarray));
    size_ = PyArray_SIZE(ndarray) * PyArray_DESCR(ndarray)->elsize;
    capacity_ = size_;

    if (PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE) {
      is_mutable_ = true;
    }
  }
}

NumPyBuffer::~NumPyBuffer() {
  PyAcquireGIL lock;
  Py_XDECREF(arr_);
}

#define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
  case NPY_##NPY_NAME:                        \
    *out = FACTORY();                         \
    break;

Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out) {
  if (!PyArray_DescrCheck(dtype)) {
    return Status::TypeError("Did not pass numpy.dtype object");
  }
  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
  int type_num = fix_numpy_type_num(descr->type_num);

  switch (type_num) {
    TO_ARROW_TYPE_CASE(BOOL, uint8);
    TO_ARROW_TYPE_CASE(INT8, int8);
    TO_ARROW_TYPE_CASE(INT16, int16);
    TO_ARROW_TYPE_CASE(INT32, int32);
    TO_ARROW_TYPE_CASE(INT64, int64);
    TO_ARROW_TYPE_CASE(UINT8, uint8);
    TO_ARROW_TYPE_CASE(UINT16, uint16);
    TO_ARROW_TYPE_CASE(UINT32, uint32);
    TO_ARROW_TYPE_CASE(UINT64, uint64);
    TO_ARROW_TYPE_CASE(FLOAT16, float16);
    TO_ARROW_TYPE_CASE(FLOAT32, float32);
    TO_ARROW_TYPE_CASE(FLOAT64, float64);
    default: {
      return Status::NotImplemented("Unsupported numpy type ", descr->type_num);
    }
  }
  return Status::OK();
}

Status GetNumPyType(const DataType& type, int* type_num) {
#define NUMPY_TYPE_CASE(ARROW_NAME, NPY_NAME) \
  case Type::ARROW_NAME:                      \
    *type_num = NPY_##NPY_NAME;               \
    break;

  switch (type.id()) {
    NUMPY_TYPE_CASE(UINT8, UINT8);
    NUMPY_TYPE_CASE(INT8, INT8);
    NUMPY_TYPE_CASE(UINT16, UINT16);
    NUMPY_TYPE_CASE(INT16, INT16);
    NUMPY_TYPE_CASE(UINT32, UINT32);
    NUMPY_TYPE_CASE(INT32, INT32);
    NUMPY_TYPE_CASE(UINT64, UINT64);
    NUMPY_TYPE_CASE(INT64, INT64);
    NUMPY_TYPE_CASE(HALF_FLOAT, FLOAT16);
    NUMPY_TYPE_CASE(FLOAT, FLOAT32);
    NUMPY_TYPE_CASE(DOUBLE, FLOAT64);
    default: {
      return Status::NotImplemented("Unsupported tensor type: ", type.ToString());
    }
  }
#undef NUMPY_TYPE_CASE

  return Status::OK();
}

Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
  if (!PyArray_DescrCheck(dtype)) {
    return Status::TypeError("Did not pass numpy.dtype object");
  }
  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
  return NumPyDtypeToArrow(descr, out);
}

Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr<DataType>* out) {
  int type_num = fix_numpy_type_num(descr->type_num);

  switch (type_num) {
    TO_ARROW_TYPE_CASE(BOOL, boolean);
    TO_ARROW_TYPE_CASE(INT8, int8);
    TO_ARROW_TYPE_CASE(INT16, int16);
    TO_ARROW_TYPE_CASE(INT32, int32);
    TO_ARROW_TYPE_CASE(INT64, int64);
    TO_ARROW_TYPE_CASE(UINT8, uint8);
    TO_ARROW_TYPE_CASE(UINT16, uint16);
    TO_ARROW_TYPE_CASE(UINT32, uint32);
    TO_ARROW_TYPE_CASE(UINT64, uint64);
    TO_ARROW_TYPE_CASE(FLOAT16, float16);
    TO_ARROW_TYPE_CASE(FLOAT32, float32);
    TO_ARROW_TYPE_CASE(FLOAT64, float64);
    TO_ARROW_TYPE_CASE(STRING, binary);
    TO_ARROW_TYPE_CASE(UNICODE, utf8);
    case NPY_DATETIME: {
      auto date_dtype =
          reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
      switch (date_dtype->meta.base) {
        case NPY_FR_s:
          *out = timestamp(TimeUnit::SECOND);
          break;
        case NPY_FR_ms:
          *out = timestamp(TimeUnit::MILLI);
          break;
        case NPY_FR_us:
          *out = timestamp(TimeUnit::MICRO);
          break;
        case NPY_FR_ns:
          *out = timestamp(TimeUnit::NANO);
          break;
        case NPY_FR_D:
          *out = date32();
          break;
        case NPY_FR_GENERIC:
          return Status::NotImplemented("Unbound or generic datetime64 time unit");
        default:
          return Status::NotImplemented("Unsupported datetime64 time unit");
      }
    } break;
    default: {
      return Status::NotImplemented("Unsupported numpy type ", descr->type_num);
    }
  }

  return Status::OK();
}

#undef TO_ARROW_TYPE_CASE

Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out) {
  PyAcquireGIL lock;

  if (!PyArray_Check(ao)) {
    return Status::TypeError("Did not pass ndarray object");
  }

  PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);

  // TODO(wesm): What do we want to do with non-contiguous memory and negative strides?

  int ndim = PyArray_NDIM(ndarray);

  std::shared_ptr<Buffer> data = std::make_shared<NumPyBuffer>(ao);
  std::vector<int64_t> shape(ndim);
  std::vector<int64_t> strides(ndim);

  npy_intp* array_strides = PyArray_STRIDES(ndarray);
  npy_intp* array_shape = PyArray_SHAPE(ndarray);
  for (int i = 0; i < ndim; ++i) {
    if (array_strides[i] < 0) {
      return Status::Invalid("Negative ndarray strides not supported");
    }
    shape[i] = array_shape[i];
    strides[i] = array_strides[i];
  }

  std::shared_ptr<DataType> type;
  RETURN_NOT_OK(
      GetTensorType(reinterpret_cast<PyObject*>(PyArray_DESCR(ndarray)), &type));
  *out = std::make_shared<Tensor>(type, data, shape, strides);
  return Status::OK();
}

Status TensorToNdarray(const std::shared_ptr<Tensor>& tensor, PyObject* base,
                       PyObject** out) {
  PyAcquireGIL lock;

  int type_num;
  RETURN_NOT_OK(GetNumPyType(*tensor->type(), &type_num));
  PyArray_Descr* dtype = PyArray_DescrNewFromType(type_num);
  RETURN_IF_PYERROR();

  const int ndim = tensor->ndim();
  std::vector<npy_intp> npy_shape(ndim);
  std::vector<npy_intp> npy_strides(ndim);

  for (int i = 0; i < ndim; ++i) {
    npy_shape[i] = tensor->shape()[i];
    npy_strides[i] = tensor->strides()[i];
  }

  const void* immutable_data = nullptr;
  if (tensor->data()) {
    immutable_data = tensor->data()->data();
  }

  // Remove const =(
  void* mutable_data = const_cast<void*>(immutable_data);

  int array_flags = 0;
  if (tensor->is_row_major()) {
    array_flags |= NPY_ARRAY_C_CONTIGUOUS;
  }
  if (tensor->is_column_major()) {
    array_flags |= NPY_ARRAY_F_CONTIGUOUS;
  }
  if (tensor->is_mutable()) {
    array_flags |= NPY_ARRAY_WRITEABLE;
  }

  PyObject* result =
      PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, npy_shape.data(),
                           npy_strides.data(), mutable_data, array_flags, nullptr);
  RETURN_IF_PYERROR()

  if (base == Py_None || base == nullptr) {
    base = py::wrap_tensor(tensor);
  } else {
    Py_XINCREF(base);
  }
  PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
  *out = result;
  return Status::OK();
}

}  // namespace py
}  // namespace arrow
<commit_msg>ARROW-4582: [Python/C++] Acquire the GIL on Py_INCREF<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/python/numpy_interop.h"

#include "arrow/python/numpy_convert.h"

#include <cstdint>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

#include "arrow/buffer.h"
#include "arrow/tensor.h"
#include "arrow/type.h"

#include "arrow/python/common.h"
#include "arrow/python/pyarrow.h"
#include "arrow/python/type_traits.h"

namespace arrow {
namespace py {

bool is_contiguous(PyObject* array) {
  if (PyArray_Check(array)) {
    return (PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) &
            (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0;
  } else {
    return false;
  }
}

NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
  PyAcquireGIL lock;
  arr_ = ao;
  Py_INCREF(ao);

  if (PyArray_Check(ao)) {
    PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);
    data_ = reinterpret_cast<const uint8_t*>(PyArray_DATA(ndarray));
    size_ = PyArray_SIZE(ndarray) * PyArray_DESCR(ndarray)->elsize;
    capacity_ = size_;

    if (PyArray_FLAGS(ndarray) & NPY_ARRAY_WRITEABLE) {
      is_mutable_ = true;
    }
  }
}

NumPyBuffer::~NumPyBuffer() {
  PyAcquireGIL lock;
  Py_XDECREF(arr_);
}

#define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \
  case NPY_##NPY_NAME:                        \
    *out = FACTORY();                         \
    break;

Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out) {
  if (!PyArray_DescrCheck(dtype)) {
    return Status::TypeError("Did not pass numpy.dtype object");
  }
  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
  int type_num = fix_numpy_type_num(descr->type_num);

  switch (type_num) {
    TO_ARROW_TYPE_CASE(BOOL, uint8);
    TO_ARROW_TYPE_CASE(INT8, int8);
    TO_ARROW_TYPE_CASE(INT16, int16);
    TO_ARROW_TYPE_CASE(INT32, int32);
    TO_ARROW_TYPE_CASE(INT64, int64);
    TO_ARROW_TYPE_CASE(UINT8, uint8);
    TO_ARROW_TYPE_CASE(UINT16, uint16);
    TO_ARROW_TYPE_CASE(UINT32, uint32);
    TO_ARROW_TYPE_CASE(UINT64, uint64);
    TO_ARROW_TYPE_CASE(FLOAT16, float16);
    TO_ARROW_TYPE_CASE(FLOAT32, float32);
    TO_ARROW_TYPE_CASE(FLOAT64, float64);
    default: {
      return Status::NotImplemented("Unsupported numpy type ", descr->type_num);
    }
  }
  return Status::OK();
}

Status GetNumPyType(const DataType& type, int* type_num) {
#define NUMPY_TYPE_CASE(ARROW_NAME, NPY_NAME) \
  case Type::ARROW_NAME:                      \
    *type_num = NPY_##NPY_NAME;               \
    break;

  switch (type.id()) {
    NUMPY_TYPE_CASE(UINT8, UINT8);
    NUMPY_TYPE_CASE(INT8, INT8);
    NUMPY_TYPE_CASE(UINT16, UINT16);
    NUMPY_TYPE_CASE(INT16, INT16);
    NUMPY_TYPE_CASE(UINT32, UINT32);
    NUMPY_TYPE_CASE(INT32, INT32);
    NUMPY_TYPE_CASE(UINT64, UINT64);
    NUMPY_TYPE_CASE(INT64, INT64);
    NUMPY_TYPE_CASE(HALF_FLOAT, FLOAT16);
    NUMPY_TYPE_CASE(FLOAT, FLOAT32);
    NUMPY_TYPE_CASE(DOUBLE, FLOAT64);
    default: {
      return Status::NotImplemented("Unsupported tensor type: ", type.ToString());
    }
  }
#undef NUMPY_TYPE_CASE

  return Status::OK();
}

Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
  if (!PyArray_DescrCheck(dtype)) {
    return Status::TypeError("Did not pass numpy.dtype object");
  }
  PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
  return NumPyDtypeToArrow(descr, out);
}

Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr<DataType>* out) {
  int type_num = fix_numpy_type_num(descr->type_num);

  switch (type_num) {
    TO_ARROW_TYPE_CASE(BOOL, boolean);
    TO_ARROW_TYPE_CASE(INT8, int8);
    TO_ARROW_TYPE_CASE(INT16, int16);
    TO_ARROW_TYPE_CASE(INT32, int32);
    TO_ARROW_TYPE_CASE(INT64, int64);
    TO_ARROW_TYPE_CASE(UINT8, uint8);
    TO_ARROW_TYPE_CASE(UINT16, uint16);
    TO_ARROW_TYPE_CASE(UINT32, uint32);
    TO_ARROW_TYPE_CASE(UINT64, uint64);
    TO_ARROW_TYPE_CASE(FLOAT16, float16);
    TO_ARROW_TYPE_CASE(FLOAT32, float32);
    TO_ARROW_TYPE_CASE(FLOAT64, float64);
    TO_ARROW_TYPE_CASE(STRING, binary);
    TO_ARROW_TYPE_CASE(UNICODE, utf8);
    case NPY_DATETIME: {
      auto date_dtype =
          reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
      switch (date_dtype->meta.base) {
        case NPY_FR_s:
          *out = timestamp(TimeUnit::SECOND);
          break;
        case NPY_FR_ms:
          *out = timestamp(TimeUnit::MILLI);
          break;
        case NPY_FR_us:
          *out = timestamp(TimeUnit::MICRO);
          break;
        case NPY_FR_ns:
          *out = timestamp(TimeUnit::NANO);
          break;
        case NPY_FR_D:
          *out = date32();
          break;
        case NPY_FR_GENERIC:
          return Status::NotImplemented("Unbound or generic datetime64 time unit");
        default:
          return Status::NotImplemented("Unsupported datetime64 time unit");
      }
    } break;
    default: {
      return Status::NotImplemented("Unsupported numpy type ", descr->type_num);
    }
  }

  return Status::OK();
}

#undef TO_ARROW_TYPE_CASE

Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* out) {
  if (!PyArray_Check(ao)) {
    return Status::TypeError("Did not pass ndarray object");
  }

  PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(ao);

  // TODO(wesm): What do we want to do with non-contiguous memory and negative strides?

  int ndim = PyArray_NDIM(ndarray);

  // This is also holding the GIL, so don't already draw it.
  std::shared_ptr<Buffer> data = std::make_shared<NumPyBuffer>(ao);
  std::vector<int64_t> shape(ndim);
  std::vector<int64_t> strides(ndim);

  {
    PyAcquireGIL lock;
    npy_intp* array_strides = PyArray_STRIDES(ndarray);
    npy_intp* array_shape = PyArray_SHAPE(ndarray);
    for (int i = 0; i < ndim; ++i) {
      if (array_strides[i] < 0) {
        return Status::Invalid("Negative ndarray strides not supported");
      }
      shape[i] = array_shape[i];
      strides[i] = array_strides[i];
    }

    std::shared_ptr<DataType> type;
    RETURN_NOT_OK(
        GetTensorType(reinterpret_cast<PyObject*>(PyArray_DESCR(ndarray)), &type));
    *out = std::make_shared<Tensor>(type, data, shape, strides);
    return Status::OK();
  }
}

Status TensorToNdarray(const std::shared_ptr<Tensor>& tensor, PyObject* base,
                       PyObject** out) {
  PyAcquireGIL lock;

  int type_num;
  RETURN_NOT_OK(GetNumPyType(*tensor->type(), &type_num));
  PyArray_Descr* dtype = PyArray_DescrNewFromType(type_num);
  RETURN_IF_PYERROR();

  const int ndim = tensor->ndim();
  std::vector<npy_intp> npy_shape(ndim);
  std::vector<npy_intp> npy_strides(ndim);

  for (int i = 0; i < ndim; ++i) {
    npy_shape[i] = tensor->shape()[i];
    npy_strides[i] = tensor->strides()[i];
  }

  const void* immutable_data = nullptr;
  if (tensor->data()) {
    immutable_data = tensor->data()->data();
  }

  // Remove const =(
  void* mutable_data = const_cast<void*>(immutable_data);

  int array_flags = 0;
  if (tensor->is_row_major()) {
    array_flags |= NPY_ARRAY_C_CONTIGUOUS;
  }
  if (tensor->is_column_major()) {
    array_flags |= NPY_ARRAY_F_CONTIGUOUS;
  }
  if (tensor->is_mutable()) {
    array_flags |= NPY_ARRAY_WRITEABLE;
  }

  PyObject* result =
      PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, npy_shape.data(),
                           npy_strides.data(), mutable_data, array_flags, nullptr);
  RETURN_IF_PYERROR()

  if (base == Py_None || base == nullptr) {
    base = py::wrap_tensor(tensor);
  } else {
    Py_XINCREF(base);
  }
  PyArray_SetBaseObject(reinterpret_cast<PyArrayObject*>(result), base);
  *out = result;
  return Status::OK();
}

}  // namespace py
}  // namespace arrow
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FirstOrderOptimizer.h"
#include "paddle/math/TrainingAlgorithmOp.h"
#include "paddle/utils/Flags.h"
#include "paddle/utils/Util.h"

#include <cmath>

DEFINE_bool(log_clipping, false, "enable log clipping or not");

namespace paddle {

SparseMomentumParameterOptimizer::SparseMomentumParameterOptimizer(
    const OptimizationConfig& optConfig)
    : ParameterOptimizer(optConfig) {
  addParameterType(PARAMETER_MOMENTUM);
  addParameterType(PARAMETER_MOMENTUM_UT);
  addParameterType(PARAMETER_MOMENTUM_VT);
  alpha_ = 1;
  beta_ = 1;
  tau_ = -1;
  threshold_ = 1e+06;
}

void SparseMomentumParameterOptimizer::init(size_t numRows,
                                            const ParameterConfig* config) {
  isParameterSparse_ = numRows != 0;
  t0Vec_.resize(numRows);
  t0Vec_.assign(t0Vec_.size(), 0);
  timer_ = 0;
  momentum_ = config->momentum();
  decayRate_ = config->decay_rate();
  gamma_ = config->learning_rate();
}

void SparseMomentumParameterOptimizer::startBatch(int64_t numSamplesProcessed) {
  learningRate_ = calcLearningRate(numSamplesProcessed, pass_);
  if (isParameterSparse_) {
    tau_ = tau_ + beta_ / alpha_;
    alpha_ = alpha_ / momentum_;
    beta_ = beta_ / (1 + decayRate_ * gamma_ * learningRate_);
  }
}

void SparseMomentumParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& paraConfig,
                                              size_t sparseId) const {
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    if (t0Vec_[sparseId] == 0) {
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
      t0Vec_[sparseId] = 1;
    }
    vecs[PARAMETER_MOMENTUM_UT]->add(*vecs[PARAMETER_GRADIENT],
                                     -alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_MOMENTUM_VT]->add(*vecs[PARAMETER_GRADIENT],
                                     tau_ * alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_VALUE]->add(*vecs[PARAMETER_MOMENTUM_UT],
                               tau_ / beta_ + 1.0 / alpha_,
                               *vecs[PARAMETER_MOMENTUM_VT],
                               1.0 / beta_);

  } else {
    vecs[PARAMETER_VALUE]->sgdUpdate(*vecs[PARAMETER_GRADIENT],
                                     *vecs[PARAMETER_MOMENTUM],
                                     learningRate_ * paraConfig.learning_rate(),
                                     paraConfig.momentum(),
                                     applyDecay_ ? paraConfig.decay_rate() : 0);
  }
}

ParameterOptimizer::TraverseCallback
SparseMomentumParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (alpha_ > threshold_ && isParameterSparse_) {
    //  Restart to avoid large value multiplication
    //  1. \alpha = 1, \beta = 1, \tau = 0
    //  2. Note that \tau * u_t + v_t = \beta \theta_t, therefore:
    //     u_t should be rescaled to u_t/alpha_
    //     v_t should be reset to \theta_t
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_MOMENTUM_UT]->divScalar(alpha_);
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
    };
  } else {
    return nullptr;
  }
}

void SparseMomentumParameterOptimizer::finishBatch() {
  timer_++;
  if (!isParameterSparse_) return;
  if (alpha_ > threshold_) {
    alpha_ = 1;
    beta_ = 1;
    tau_ = -1;
  }
}

void AdagradParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum_buffer = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adagradApply(value,
               grad,
               mom,
               accum_buffer,
               accum,
               lr,
               epsilon,
               learningRate,
               momentum,
               decayRate);
}

ParameterOptimizer::TraverseCallback
AdagradParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (numUpdates_ % kMaxNumAccumulates == 0) {
    // Move the sum to a different buffer to avoid loss of precision
    // due to too many sums.
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_GRADIENT_SQURESUM]->add(
          *vecs[PARAMETER_GRADIENT_SQURESUM1]);
      vecs[PARAMETER_GRADIENT_SQURESUM1]->zeroMem();
    };
  } else {
    return nullptr;
  }
}

void AdaDeltaParameterOptimizer::update(const VectorPtr vecs[],
                                        const ParameterConfig& config,
                                        size_t sparseId) const {
  CHECK(sparseId == -1LU) << "Sparse update is not supported";

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum_update = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adadeltaApply(value,
                grad,
                mom,
                accum,
                accum_update,
                lr,
                rou_,
                epsilon_,
                learningRate,
                momentum,
                decayRate);
}

void RMSPropParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& sum1 = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  rmspropApply(value,
               grad,
               mom,
               sum,
               sum1,
               lr,
               accumulatedRou,
               rou_,
               epsilon,
               learningRate,
               momentum,
               decayRate,
               firstTime);
}

void DecayedAdagradParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& config,
                                              size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  decayedAdagradApply(value,
                      grad,
                      mom,
                      sum,
                      lr,
                      accumulatedRou,
                      rou_,
                      epsilon,
                      learningRate,
                      momentum,
                      decayRate,
                      firstTime);
}

void AdamParameterOptimizer::update(const VectorPtr vecs[],
                                    const ParameterConfig& config,
                                    size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";

  real beta1_power = std::pow(beta1_, step_);
  real beta2_power = std::pow(beta2_, step_);
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& v = *vecs[PARAMETER_SECOND_MOMENTUM];

  adamApply(value,
            grad,
            mom,
            v,
            beta1_,
            beta2_,
            beta1_power,
            beta2_power,
            epsilon_,
            learningRate);
}

void AdamaxParameterOptimizer::update(const VectorPtr vecs[],
                                      const ParameterConfig& config,
                                      size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& u = *vecs[PARAMETER_WEIGHTED_INFINITY_NORM];

  adamaxApply(value, grad, mom, u, beta1_, beta2_, step_, learningRate);
}

void OptimizerWithGradientClipping::update(const VectorPtr vecs[],
                                           const ParameterConfig& config,
                                           size_t sparseId) const {
  real globalThreshold = optConfig_.gradient_clipping_threshold();
  real localThreshold = config.gradient_clipping_threshold();

  real threshold;
  std::string field;
  // Get the minimum of local and global threshold
  // as the real threshold for clipping
  if (globalThreshold > 0.0f && localThreshold > 0.0f) {
    threshold =
        globalThreshold < localThreshold ? globalThreshold : localThreshold;
    field = globalThreshold < localThreshold ? "global" : "local";
  } else if (globalThreshold > 0.0f) {
    threshold = globalThreshold;
    field = "global";
  } else {
    threshold = localThreshold;
    field = "local";
  }

  real maxAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsMax();
  if (maxAbsGrad > threshold) {
    if (FLAGS_log_clipping) {
      real avgAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsSum() /
                        vecs[PARAMETER_GRADIENT]->getSize();
      LOG(INFO) << "parameter=" << config.name() << " need clipping by "
                << field << " threshold=" << threshold
                << ", max grad=" << maxAbsGrad << ", avg grad=" << avgAbsGrad;
    }
    vecs[PARAMETER_GRADIENT]->clip(-threshold, threshold);
  }
  optimizer_->update(vecs, config, sparseId);
}

}  // namespace paddle
<commit_msg>change the way of setting threshold<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FirstOrderOptimizer.h"
#include "paddle/math/TrainingAlgorithmOp.h"
#include "paddle/utils/Flags.h"
#include "paddle/utils/Util.h"

#include <cmath>

DEFINE_bool(log_clipping, false, "enable log clipping or not");

namespace paddle {

SparseMomentumParameterOptimizer::SparseMomentumParameterOptimizer(
    const OptimizationConfig& optConfig)
    : ParameterOptimizer(optConfig) {
  addParameterType(PARAMETER_MOMENTUM);
  addParameterType(PARAMETER_MOMENTUM_UT);
  addParameterType(PARAMETER_MOMENTUM_VT);
  alpha_ = 1;
  beta_ = 1;
  tau_ = -1;
  threshold_ = 1e+06;
}

void SparseMomentumParameterOptimizer::init(size_t numRows,
                                            const ParameterConfig* config) {
  isParameterSparse_ = numRows != 0;
  t0Vec_.resize(numRows);
  t0Vec_.assign(t0Vec_.size(), 0);
  timer_ = 0;
  momentum_ = config->momentum();
  decayRate_ = config->decay_rate();
  gamma_ = config->learning_rate();
}

void SparseMomentumParameterOptimizer::startBatch(int64_t numSamplesProcessed) {
  learningRate_ = calcLearningRate(numSamplesProcessed, pass_);
  if (isParameterSparse_) {
    tau_ = tau_ + beta_ / alpha_;
    alpha_ = alpha_ / momentum_;
    beta_ = beta_ / (1 + decayRate_ * gamma_ * learningRate_);
  }
}

void SparseMomentumParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& paraConfig,
                                              size_t sparseId) const {
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    if (t0Vec_[sparseId] == 0) {
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
      t0Vec_[sparseId] = 1;
    }
    vecs[PARAMETER_MOMENTUM_UT]->add(*vecs[PARAMETER_GRADIENT],
                                     -alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_MOMENTUM_VT]->add(*vecs[PARAMETER_GRADIENT],
                                     tau_ * alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_VALUE]->add(*vecs[PARAMETER_MOMENTUM_UT],
                               tau_ / beta_ + 1.0 / alpha_,
                               *vecs[PARAMETER_MOMENTUM_VT],
                               1.0 / beta_);

  } else {
    vecs[PARAMETER_VALUE]->sgdUpdate(*vecs[PARAMETER_GRADIENT],
                                     *vecs[PARAMETER_MOMENTUM],
                                     learningRate_ * paraConfig.learning_rate(),
                                     paraConfig.momentum(),
                                     applyDecay_ ? paraConfig.decay_rate() : 0);
  }
}

ParameterOptimizer::TraverseCallback
SparseMomentumParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (alpha_ > threshold_ && isParameterSparse_) {
    //  Restart to avoid large value multiplication
    //  1. \alpha = 1, \beta = 1, \tau = 0
    //  2. Note that \tau * u_t + v_t = \beta \theta_t, therefore:
    //     u_t should be rescaled to u_t/alpha_
    //     v_t should be reset to \theta_t
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_MOMENTUM_UT]->divScalar(alpha_);
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
    };
  } else {
    return nullptr;
  }
}

void SparseMomentumParameterOptimizer::finishBatch() {
  timer_++;
  if (!isParameterSparse_) return;
  if (alpha_ > threshold_) {
    alpha_ = 1;
    beta_ = 1;
    tau_ = -1;
  }
}

void AdagradParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum_buffer = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adagradApply(value,
               grad,
               mom,
               accum_buffer,
               accum,
               lr,
               epsilon,
               learningRate,
               momentum,
               decayRate);
}

ParameterOptimizer::TraverseCallback
AdagradParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (numUpdates_ % kMaxNumAccumulates == 0) {
    // Move the sum to a different buffer to avoid loss of precision
    // due to too many sums.
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_GRADIENT_SQURESUM]->add(
          *vecs[PARAMETER_GRADIENT_SQURESUM1]);
      vecs[PARAMETER_GRADIENT_SQURESUM1]->zeroMem();
    };
  } else {
    return nullptr;
  }
}

void AdaDeltaParameterOptimizer::update(const VectorPtr vecs[],
                                        const ParameterConfig& config,
                                        size_t sparseId) const {
  CHECK(sparseId == -1LU) << "Sparse update is not supported";

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum_update = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adadeltaApply(value,
                grad,
                mom,
                accum,
                accum_update,
                lr,
                rou_,
                epsilon_,
                learningRate,
                momentum,
                decayRate);
}

void RMSPropParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& sum1 = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  rmspropApply(value,
               grad,
               mom,
               sum,
               sum1,
               lr,
               accumulatedRou,
               rou_,
               epsilon,
               learningRate,
               momentum,
               decayRate,
               firstTime);
}

void DecayedAdagradParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& config,
                                              size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  decayedAdagradApply(value,
                      grad,
                      mom,
                      sum,
                      lr,
                      accumulatedRou,
                      rou_,
                      epsilon,
                      learningRate,
                      momentum,
                      decayRate,
                      firstTime);
}

void AdamParameterOptimizer::update(const VectorPtr vecs[],
                                    const ParameterConfig& config,
                                    size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";

  real beta1_power = std::pow(beta1_, step_);
  real beta2_power = std::pow(beta2_, step_);
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& v = *vecs[PARAMETER_SECOND_MOMENTUM];

  adamApply(value,
            grad,
            mom,
            v,
            beta1_,
            beta2_,
            beta1_power,
            beta2_power,
            epsilon_,
            learningRate);
}

void AdamaxParameterOptimizer::update(const VectorPtr vecs[],
                                      const ParameterConfig& config,
                                      size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& u = *vecs[PARAMETER_WEIGHTED_INFINITY_NORM];

  adamaxApply(value, grad, mom, u, beta1_, beta2_, step_, learningRate);
}

void OptimizerWithGradientClipping::update(const VectorPtr vecs[],
                                           const ParameterConfig& config,
                                           size_t sparseId) const {
  real globalThreshold = optConfig_.gradient_clipping_threshold();
  real localThreshold = config.gradient_clipping_threshold();

  real threshold;
  std::string field;
  // Use local gradient clipping threshold if it's enabled,
  // otherwise using the global one.
  if (localThreshold > 0.0f) {
    threshold = localThreshold;
    field = "local";
  } else {
    threshold = globalThreshold;
    field = "global";
  }

  real maxAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsMax();
  if (maxAbsGrad > threshold) {
    if (FLAGS_log_clipping) {
      real avgAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsSum() /
                        vecs[PARAMETER_GRADIENT]->getSize();
      LOG(INFO) << "parameter=" << config.name() << " need clipping by "
                << field << " threshold=" << threshold
                << ", max grad=" << maxAbsGrad << ", avg grad=" << avgAbsGrad;
    }
    vecs[PARAMETER_GRADIENT]->clip(-threshold, threshold);
  }
  optimizer_->update(vecs, config, sparseId);
}

}  // namespace paddle
<|endoftext|>


<commit_before>// Filename: rocketFileInterface.cxx
// Created by:  rdb (03Nov11)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) Carnegie Mellon University.  All rights reserved.
//
// All use of this software is subject to the terms of the revised BSD
// license.  You should have received a copy of this license along
// with this source code in a file named "LICENSE."
//
////////////////////////////////////////////////////////////////////

#include "rocketFileInterface.h"
#include "virtualFileSystem.h"

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Constructor
//       Access: Public
//  Description: Constructs a RocketFileInterface for the given
//               VFS, or the default if NULL is given.
////////////////////////////////////////////////////////////////////
RocketFileInterface::
RocketFileInterface(VirtualFileSystem *vfs) : _vfs(vfs) {
  if (_vfs == NULL) {
    _vfs = VirtualFileSystem::get_global_ptr();
  }
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Open
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Rocket::Core::FileHandle RocketFileInterface::
Open(const Rocket::Core::String& path) {
  rocket_cat.debug() << "Opening " << path.CString() << "\n";

  // A FileHandle is actually just a void pointer
  return (Rocket::Core::FileHandle)
    _vfs->open_read_file(Filename::from_os_specific(path.CString()), true);
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Close
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
void RocketFileInterface::
Close(Rocket::Core::FileHandle file) {
  if ((istream*) file != (istream*) NULL) {
    _vfs->close_read_file((istream*) file);
  }
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Read
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
size_t RocketFileInterface::
Read(void* buffer, size_t size, Rocket::Core::FileHandle file) {
  istream* const stream = (istream*) file;
  if (stream == (istream*) NULL) {
    return 0;
  }

  stream->read((char*) buffer, size);
  return stream->gcount();
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Seek
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
bool RocketFileInterface::
Seek(Rocket::Core::FileHandle file, long offset, int origin) {
  istream* stream = (istream*) file;
  if (stream == (istream*) NULL) {
    return false;
  }

  switch(origin) {
  case SEEK_SET:
    stream->seekg(offset, ios::beg);
    break;
  case SEEK_CUR:
    stream->seekg(offset, ios::cur);
    break;
  case SEEK_END:
    stream->seekg(offset, ios::end);
  };

  return !stream->fail();
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Tell
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
size_t RocketFileInterface::
Tell(Rocket::Core::FileHandle file) {
  if ((istream*) file == (istream*) NULL) {
    return -1;
  }
  return ((istream*) file)->tellg();
}
<commit_msg>show error message when rocket fails to open a file<commit_after>// Filename: rocketFileInterface.cxx
// Created by:  rdb (03Nov11)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) Carnegie Mellon University.  All rights reserved.
//
// All use of this software is subject to the terms of the revised BSD
// license.  You should have received a copy of this license along
// with this source code in a file named "LICENSE."
//
////////////////////////////////////////////////////////////////////

#include "rocketFileInterface.h"
#include "virtualFileSystem.h"

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Constructor
//       Access: Public
//  Description: Constructs a RocketFileInterface for the given
//               VFS, or the default if NULL is given.
////////////////////////////////////////////////////////////////////
RocketFileInterface::
RocketFileInterface(VirtualFileSystem *vfs) : _vfs(vfs) {
  if (_vfs == NULL) {
    _vfs = VirtualFileSystem::get_global_ptr();
  }
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Open
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Rocket::Core::FileHandle RocketFileInterface::
Open(const Rocket::Core::String& path) {
  rocket_cat.debug() << "Opening " << path.CString() << "\n";

  Filename fn = Filename::from_os_specific(path.CString());
  void *ptr = (void*) _vfs->open_read_file(fn, true);

  if (ptr == NULL) {
    rocket_cat.error() << "Failed to open " << fn << "\n";
  }

  // A FileHandle is actually just a void pointer
  return (Rocket::Core::FileHandle) ptr;
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Close
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
void RocketFileInterface::
Close(Rocket::Core::FileHandle file) {
  if ((istream*) file != (istream*) NULL) {
    _vfs->close_read_file((istream*) file);
  }
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Read
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
size_t RocketFileInterface::
Read(void* buffer, size_t size, Rocket::Core::FileHandle file) {
  istream* const stream = (istream*) file;
  if (stream == (istream*) NULL) {
    return 0;
  }

  stream->read((char*) buffer, size);
  return stream->gcount();
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Seek
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
bool RocketFileInterface::
Seek(Rocket::Core::FileHandle file, long offset, int origin) {
  istream* stream = (istream*) file;
  if (stream == (istream*) NULL) {
    return false;
  }

  switch(origin) {
  case SEEK_SET:
    stream->seekg(offset, ios::beg);
    break;
  case SEEK_CUR:
    stream->seekg(offset, ios::cur);
    break;
  case SEEK_END:
    stream->seekg(offset, ios::end);
  };

  return !stream->fail();
}

////////////////////////////////////////////////////////////////////
//     Function: RocketFileInterface::Tell
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
size_t RocketFileInterface::
Tell(Rocket::Core::FileHandle file) {
  if ((istream*) file == (istream*) NULL) {
    return -1;
  }
  return ((istream*) file)->tellg();
}
<|endoftext|>