<commit_before>#ifndef K3_RUNTIME_COMMON_H
#define K3_RUNTIME_COMMON_H

#include <list>
#include <map>
#include <memory>
#include <stdexcept>
#include <tuple>
#include <utility>
#include <boost/any.hpp>
#include <boost/asio.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/log/core.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <boost/log/sources/severity_channel_logger.hpp>
#include <boost/log/sources/severity_feature.hpp>
#include <boost/log/trivial.hpp>
#include <boost/phoenix/core.hpp>
#include <boost/phoenix/stl/algorithm.hpp>

namespace K3 {

  using namespace std;
  using boost::any;

  using namespace boost::log;
  using namespace boost::log::sources;
  using namespace boost::log::trivial;
  using namespace boost::phoenix;

  typedef string Identifier;

  typedef tuple<boost::asio::ip::address, unsigned short> Address;

  enum class Builtin { Stdin, Stdout, Stderr };
  enum class IOMode  { Read, Write, Append, ReadWrite };

  //---------------
  // Addresses.

  Address make_address(const string& host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  Address make_address(const char* host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  Address make_address(const string&& host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  inline string addressHost(const Address& addr) { return get<0>(addr).to_string(); }
  inline string addressHost(Address&& addr) { return get<0>(std::forward<Address>(addr)).to_string(); }

  inline int addressPort(const Address& addr) { return get<1>(addr); }
  inline int addressPort(Address&& addr) { return get<1>(std::forward<Address>(addr)); }

  string addressAsString(const Address& addr) {
    return addressHost(addr) + ":" + to_string(addressPort(addr));
  }

  string addressAsString(Address&& addr) {
    return addressHost(std::forward<Address>(addr))
            + ":" + to_string(addressPort(std::forward<Address>(addr)));
  }

  Address internalSendAddress(const Address& addr) {
    return make_address(addressHost(addr), addressPort(addr)+1);
  }

  Address internalSendAddress(Address&& addr) {
    return make_address(addressHost(std::forward<Address>(addr)),
                        addressPort(std::forward<Address>(addr))+1);
  }

  Address externalSendAddress(const Address& addr) {
    return make_address(addressHost(addr), addressPort(addr)+2);
  }

  Address externalSendAddress(Address&& addr) {
    return make_address(addressHost(std::forward<Address>(addr)),
                        addressPort(std::forward<Address>(addr))+2);
  }

  Address defaultAddress = make_address("127.0.0.1", 40000);

  //-------------
  // Messages.

  template<typename Value>
  class Message : public tuple<Address, Identifier, Value> {
  public:
    Message(Address addr, Identifier id, const Value& v)
      : tuple<Address, Identifier, Value>(std::move(addr), std::move(id), v)
    {}

    Message(Address addr, Identifier id, Value&& v)
      : tuple<Address, Identifier, Value>(std::move(addr), std::move(id), std::forward<Value>(v))
    {}

    Message(Address&& addr, Identifier&& id, Value&& v)
      : tuple<Address, Identifier, Value>(std::forward<Address>(addr),
                                          std::forward<Identifier>(id),
                                          std::forward<Value>(v))
    {}

    Address&    address()  { return get<0>(*this); }
    Identifier& id()       { return get<1>(*this); }
    Value&      contents() { return get<2>(*this); }
    string      target()   { return id() + "@" + addressAsString(address()); }
  };

  //--------------------
  // System environment.

  // Literals are native values rather than an AST reprensentation as in Haskell.
  typedef any Literal;
  typedef map<Identifier, Literal> PeerBootstrap;
  typedef map<Address, PeerBootstrap> SystemEnvironment;

  list<Address> deployedNodes(const SystemEnvironment& sysEnv) {
    list<Address> r;
    for ( auto x : sysEnv ) { r.push_back(x.first); }
    return std::move(r);
  }

  bool isDeployedNode(const SystemEnvironment& sysEnv, Address addr) {
    return sysEnv.find(addr) != sysEnv.end();
  }

  //-------------
  // Logging.
  class Log {
  public:
    Log() {}
    Log(severity_level lvl) : defaultLevel(lvl) {}

    virtual void log(const string& msg) = 0;
    virtual void log(const char* msg) = 0;
    virtual void logAt(severity_level lvl, const string& msg) = 0;
    virtual void logAt(severity_level lvl, const char* msg) = 0;

  protected:
    severity_level defaultLevel;
  };

  class LogST : public severity_channel_logger<severity_level,string>, public Log
  {
  public:
    typedef severity_channel_logger<severity_level,string> logger;

    LogST(string chan) : logger(keywords::channel = chan), Log(severity_level::info) {}
    LogST(string chan, severity_level lvl) : logger(keywords::channel = chan), Log(lvl) {}

    void log(const string& msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }
    void log(const char* msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }

    void logAt(severity_level lvl, const string& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
    void logAt(severity_level lvl, const char& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
  };

  class LogMT : public severity_channel_logger_mt<severity_level,string>, public Log
  {
  public:
    typedef severity_channel_logger_mt<severity_level,string> logger;

    LogMT(string chan) : logger(keywords::channel = chan), Log(severity_level::info) {}
    LogMT(string chan, severity_level lvl) : logger(keywords::channel = chan), Log(lvl) {}

    void log(const string& msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }
    void log(const char* msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }

    void logAt(severity_level lvl, const string& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
    void logAt(severity_level lvl, const char* msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
  };

  //--------------------
  // Wire descriptions

  // A generic exception that can be thrown by wire descriptor methods.
  class WireDescException : public runtime_error {
  public:
    WireDescException(const string& msg) : runtime_error(msg) {}
    WireDescException(const char* msg) : runtime_error(msg) {}
  };

  // Message serializtion/deserialization abstract base class.
  // Implementations can encapsulate framing concerns as well as serdes operations.
  //
  // The unpack method may be supplied a complete or incomplete string corresponding
  // to a value. It is left to the implementation to determine the scope of functionality
  // supported, for example partial unpacking (e.g., for network sockets).
  // The semantics of repeated invocations are dependent on the actual implementation
  // of the wire description (including factors such as message loss).
  // This includes the conditions under which an exception is thrown.
  template<typename T>
  class WireDesc : public virtual LogMT {
    public:
      WireDesc() : LogMT("WireDesc") {}
      virtual string pack(const T& payload) = 0;
      virtual shared_ptr<T> unpack(const string& message) = 0;
  };

  class DefaultWireDesc : public WireDesc<string> {
    public:
      DefaultWireDesc() : LogMT("DefaultWireDesc") {}
      string pack(const string& payload) { return payload; }
      shared_ptr<string> unpack(const string& message) { return shared_ptr<string>(new string(message)); }
  };

  template <class T>
  class BoostWireDesc : public virtual LogMT {
    public:
      BoostWireDesc() : LogMT("BoostWireDesc") {}

      static string pack(const T& payload) {
          ostringstream out_sstream;
          boost::archive::text_oarchive out_archive(out_sstream);
          out_archive << payload;
          return out_sstream.str();
      }

      static shared_ptr<T> unpack(const string& message) {
          istringstream in_sstream(message);
          boost::archive::text_iarchive in_archive(in_sstream);

          shared_ptr<T> p;
          in_archive >> *p;
          return p;
      }
  };

  /*
  template <template<class> class F>
  class WireDesc : public virtual LogMT {
      public:
          WireDesc() : LogMT("WireDesc") {}

          template <class T>
          string pack(const T& payload) {
              return F<T>::pack(payload);
          }

          template <class T>
          shared_ptr<T> unpack(const string& message) {
              return F<T>::unpack(message);
          }
  };

  template <class T>
  class BoostWireDesc : public WireDesc<BoostWireDesc> {
      public:
          BoostWireDesc() : WireDesc(), LogMT("BoostWireDesc") {}

          static string pack(const T& payload) {
              ostringstream out_sstream;
              boost::archive::text_oarchive out_archive(out_sstream);
              out_archive << payload;
              return out_sstream.str();
          }

          static shared_ptr<T> unpack(const string& message) {
              istringstream in_sstream(message);
              boost::archive::text_iarchive in_archive(in_sstream);

              shared_ptr<T> p;
              in_archive >> *p;
              return p;
          }
  };
  */

  // TODO: protobuf, msgpack, json WireDesc implementations.

}

#endif
<commit_msg>Add const methods to return const ref in Message<commit_after>#ifndef K3_RUNTIME_COMMON_H
#define K3_RUNTIME_COMMON_H

#include <list>
#include <map>
#include <memory>
#include <stdexcept>
#include <tuple>
#include <utility>
#include <boost/any.hpp>
#include <boost/asio.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/log/core.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <boost/log/sources/severity_channel_logger.hpp>
#include <boost/log/sources/severity_feature.hpp>
#include <boost/log/trivial.hpp>
#include <boost/phoenix/core.hpp>
#include <boost/phoenix/stl/algorithm.hpp>

namespace K3 {

  using namespace std;
  using boost::any;

  using namespace boost::log;
  using namespace boost::log::sources;
  using namespace boost::log::trivial;
  using namespace boost::phoenix;

  typedef string Identifier;

  typedef tuple<boost::asio::ip::address, unsigned short> Address;

  enum class Builtin { Stdin, Stdout, Stderr };
  enum class IOMode  { Read, Write, Append, ReadWrite };

  //---------------
  // Addresses.

  Address make_address(const string& host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  Address make_address(const char* host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  Address make_address(const string&& host, unsigned short port) {
    return Address(boost::asio::ip::address::from_string(host), port);
  }

  inline string addressHost(const Address& addr) { return get<0>(addr).to_string(); }
  inline string addressHost(Address&& addr) { return get<0>(std::forward<Address>(addr)).to_string(); }

  inline int addressPort(const Address& addr) { return get<1>(addr); }
  inline int addressPort(Address&& addr) { return get<1>(std::forward<Address>(addr)); }

  string addressAsString(const Address& addr) {
    return addressHost(addr) + ":" + to_string(addressPort(addr));
  }

  string addressAsString(Address&& addr) {
    return addressHost(std::forward<Address>(addr))
            + ":" + to_string(addressPort(std::forward<Address>(addr)));
  }

  Address internalSendAddress(const Address& addr) {
    return make_address(addressHost(addr), addressPort(addr)+1);
  }

  Address internalSendAddress(Address&& addr) {
    return make_address(addressHost(std::forward<Address>(addr)),
                        addressPort(std::forward<Address>(addr))+1);
  }

  Address externalSendAddress(const Address& addr) {
    return make_address(addressHost(addr), addressPort(addr)+2);
  }

  Address externalSendAddress(Address&& addr) {
    return make_address(addressHost(std::forward<Address>(addr)),
                        addressPort(std::forward<Address>(addr))+2);
  }

  Address defaultAddress = make_address("127.0.0.1", 40000);

  //-------------
  // Messages.

  template<typename Value>
  class Message : public tuple<Address, Identifier, Value> {
  public:
    Message(Address addr, Identifier id, const Value& v)
      : tuple<Address, Identifier, Value>(std::move(addr), std::move(id), v)
    {}

    Message(Address addr, Identifier id, Value&& v)
      : tuple<Address, Identifier, Value>(std::move(addr), std::move(id), std::forward<Value>(v))
    {}

    Message(Address&& addr, Identifier&& id, Value&& v)
      : tuple<Address, Identifier, Value>(std::forward<Address>(addr),
                                          std::forward<Identifier>(id),
                                          std::forward<Value>(v))
    {}

    Address&    address()  { return get<0>(*this); }
    Identifier& id()       { return get<1>(*this); }
    Value&      contents() { return get<2>(*this); }
    string      target()   { return id() + "@" + addressAsString(address()); }
    const Address&    address()  const { return get<0>(*this); }
    const Identifier& id()       const { return get<1>(*this); }
    const Value&      contents() const { return get<2>(*this); }
    const string      target()   const { return id() + "@" + addressAsString(address()); }
  };

  //--------------------
  // System environment.

  // Literals are native values rather than an AST reprensentation as in Haskell.
  typedef any Literal;
  typedef map<Identifier, Literal> PeerBootstrap;
  typedef map<Address, PeerBootstrap> SystemEnvironment;

  list<Address> deployedNodes(const SystemEnvironment& sysEnv) {
    list<Address> r;
    for ( auto x : sysEnv ) { r.push_back(x.first); }
    return std::move(r);
  }

  bool isDeployedNode(const SystemEnvironment& sysEnv, Address addr) {
    return sysEnv.find(addr) != sysEnv.end();
  }

  //-------------
  // Logging.
  class Log {
  public:
    Log() {}
    Log(severity_level lvl) : defaultLevel(lvl) {}

    virtual void log(const string& msg) = 0;
    virtual void log(const char* msg) = 0;
    virtual void logAt(severity_level lvl, const string& msg) = 0;
    virtual void logAt(severity_level lvl, const char* msg) = 0;

  protected:
    severity_level defaultLevel;
  };

  class LogST : public severity_channel_logger<severity_level,string>, public Log
  {
  public:
    typedef severity_channel_logger<severity_level,string> logger;

    LogST(string chan) : logger(keywords::channel = chan), Log(severity_level::info) {}
    LogST(string chan, severity_level lvl) : logger(keywords::channel = chan), Log(lvl) {}

    void log(const string& msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }
    void log(const char* msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }

    void logAt(severity_level lvl, const string& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
    void logAt(severity_level lvl, const char& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
  };

  class LogMT : public severity_channel_logger_mt<severity_level,string>, public Log
  {
  public:
    typedef severity_channel_logger_mt<severity_level,string> logger;

    LogMT(string chan) : logger(keywords::channel = chan), Log(severity_level::info) {}
    LogMT(string chan, severity_level lvl) : logger(keywords::channel = chan), Log(lvl) {}

    void log(const string& msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }
    void log(const char* msg) { BOOST_LOG_SEV(*this, defaultLevel) << msg; }

    void logAt(severity_level lvl, const string& msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
    void logAt(severity_level lvl, const char* msg) { BOOST_LOG_SEV(*this, lvl) << msg; }
  };

  //--------------------
  // Wire descriptions

  // A generic exception that can be thrown by wire descriptor methods.
  class WireDescException : public runtime_error {
  public:
    WireDescException(const string& msg) : runtime_error(msg) {}
    WireDescException(const char* msg) : runtime_error(msg) {}
  };

  // Message serializtion/deserialization abstract base class.
  // Implementations can encapsulate framing concerns as well as serdes operations.
  //
  // The unpack method may be supplied a complete or incomplete string corresponding
  // to a value. It is left to the implementation to determine the scope of functionality
  // supported, for example partial unpacking (e.g., for network sockets).
  // The semantics of repeated invocations are dependent on the actual implementation
  // of the wire description (including factors such as message loss).
  // This includes the conditions under which an exception is thrown.
  template<typename T>
  class WireDesc : public virtual LogMT {
    public:
      WireDesc() : LogMT("WireDesc") {}
      virtual string pack(const T& payload) = 0;
      virtual shared_ptr<T> unpack(const string& message) = 0;
  };

  class DefaultWireDesc : public WireDesc<string> {
    public:
      DefaultWireDesc() : LogMT("DefaultWireDesc") {}
      string pack(const string& payload) { return payload; }
      shared_ptr<string> unpack(const string& message) { return shared_ptr<string>(new string(message)); }
  };

  template <class T>
  class BoostWireDesc : public virtual LogMT {
    public:
      BoostWireDesc() : LogMT("BoostWireDesc") {}

      static string pack(const T& payload) {
          ostringstream out_sstream;
          boost::archive::text_oarchive out_archive(out_sstream);
          out_archive << payload;
          return out_sstream.str();
      }

      static shared_ptr<T> unpack(const string& message) {
          istringstream in_sstream(message);
          boost::archive::text_iarchive in_archive(in_sstream);

          shared_ptr<T> p;
          in_archive >> *p;
          return p;
      }
  };

  /*
  template <template<class> class F>
  class WireDesc : public virtual LogMT {
      public:
          WireDesc() : LogMT("WireDesc") {}

          template <class T>
          string pack(const T& payload) {
              return F<T>::pack(payload);
          }

          template <class T>
          shared_ptr<T> unpack(const string& message) {
              return F<T>::unpack(message);
          }
  };

  template <class T>
  class BoostWireDesc : public WireDesc<BoostWireDesc> {
      public:
          BoostWireDesc() : WireDesc(), LogMT("BoostWireDesc") {}

          static string pack(const T& payload) {
              ostringstream out_sstream;
              boost::archive::text_oarchive out_archive(out_sstream);
              out_archive << payload;
              return out_sstream.str();
          }

          static shared_ptr<T> unpack(const string& message) {
              istringstream in_sstream(message);
              boost::archive::text_iarchive in_archive(in_sstream);

              shared_ptr<T> p;
              in_archive >> *p;
              return p;
          }
  };
  */

  // TODO: protobuf, msgpack, json WireDesc implementations.

}

#endif
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * Copyright (c) 2019 by Contributors
 * \file eliminate_common_expr.cc
 * \brief Eliminate common expressions in the graph
 * \author Przemyslaw Tredak
 */

#include <mxnet/base.h>
#include <mxnet/op_attr_types.h>

#include <vector>
#include <map>
#include <utility>
#include <sstream>

namespace mxnet {
namespace exec {

namespace {

using nnvm::Node;
using nnvm::ObjectPtr;
using nnvm::Graph;
using nnvm::IndexedGraph;

// NodeInput holds the sufficient subset of NodeEntry fields for Node-input equality tests
using NodeInput = std::pair<const Node*, uint32_t>;

/*!
 * \brief Convert a Node's input vector of `NodeEntry` to a vector of the simpler `NodeInput`
 */
std::vector<NodeInput> ConvertInputs(const std::vector<nnvm::NodeEntry>& inputs) {
  std::vector<NodeInput> ret;
  ret.reserve(inputs.size());
  for (const auto& entry : inputs) {
    ret.emplace_back(entry.node.get(), entry.index);
  }
  return ret;
}

/*!
 * \brief Determine if two Nodes have equal function such that one Node can be eliminated.
 */
bool NodeEqual(const Node* n, const Node* m) {
  if (n->is_variable() || m->is_variable()) return false;
  if (n->op() != m->op()) return false;
  // Nodes with different attributes are considered not identical,
  // though this may reject Node pairs that are in fact functionally the same.
  if (n->attrs.dict != m->attrs.dict) return false;

  // Ops that mutate inputs cannot be optimized out
  static auto& fmutate_inputs = Op::GetAttr<nnvm::FMutateInputs>("FMutateInputs");
  if (fmutate_inputs.get(n->op(), nullptr) != nullptr) return false;

  // Stateful ops cannot be be equal to each other
  static auto& fstateful = Op::GetAttr<FCreateOpState>("FCreateOpState");
  if (fstateful.get(n->op(), nullptr) != nullptr)
    return false;

  // Check to see if the user has explicitly set THasDeterministicOutput to override the
  // subsequent determination of Node equality based on resource use.
  static auto& deterministic_output =
      Op::GetAttr<THasDeterministicOutput>("THasDeterministicOutput");
  if (deterministic_output.contains(n->op()))
    return deterministic_output[n->op()];

  // Ops that require resource could ask for
  // random resource, so need to be explicitly marked
  // to be eligible
  static auto& resource_request = Op::GetAttr<FResourceRequest>("FResourceRequest");
  static auto& resource_request_ex = Op::GetAttr<FResourceRequestEx>("FResourceRequestEx");
  if (resource_request.get(n->op(), nullptr) != nullptr) return false;
  if (resource_request_ex.get(n->op(), nullptr) != nullptr) return false;

  return true;
}

// Graph traversal to create a list of pairs of identical-function nodes that can be combined.
std::vector<std::pair<ObjectPtr, ObjectPtr> > GetCommonNodes(const Graph& g) {
  std::vector<std::pair<ObjectPtr, ObjectPtr> > ret;
  // A map between a vector of inputs and those nodes that have those inputs
  std::map<std::vector<NodeInput>, std::vector<const ObjectPtr*> > grouped_nodes;
  // Traverse the graph and group the nodes by their vector of inputs
  nnvm::DFSVisit(g.outputs, [&grouped_nodes](const ObjectPtr& n) {
    if (n->inputs.size() != 0) {
      grouped_nodes[ConvertInputs(n->inputs)].push_back(&n);
    }
  });
  // Now check for identical node ops within the node groups (having identical inputs)
  for (const auto& pair : grouped_nodes) {
    auto &node_group = pair.second;  // Group of nodes that share the same vector of inputs
    if (node_group.size() > 1) {
      std::unordered_set<size_t> visited;
      for (size_t i = 0; i < node_group.size(); ++i) {
        if (visited.count(i)) continue;
        for (size_t j = i + 1; j < node_group.size(); ++j) {
          // If the two Nodes have equal function, then one Node (called the 'replaced') can
          // be eliminated in favor of the other Node (the 'src').
          if (NodeEqual(node_group[i]->get(), node_group[j]->get())) {
            visited.insert(j);
            ObjectPtr src = *node_group[i];
            ObjectPtr replaced = *node_group[j];
            ret.emplace_back(src, replaced);
          }
        }
      }
    }
  }
  return ret;
}

/*!
 * \brief Do a single pass of Node elimination given pairs of identical Nodes.
 */
void EliminateCommonNodes(Graph* g,
                          const std::vector<std::pair<ObjectPtr, ObjectPtr> >& common_nodes) {
  for (const auto &p : common_nodes) {
    std::vector <ObjectPtr> nodes_to_change;
    const ObjectPtr &src = p.first;
    const ObjectPtr &replaced = p.second;
    // Create a `nodes_to_change` list containing the Nodes that refer to the `replaced` Node
    // that is targeted for elimination.
    DFSVisit(g->outputs, [replaced, &nodes_to_change](const ObjectPtr &n) {
      for (const auto &dep : n->control_deps) {
        if (dep == replaced) {
          nodes_to_change.push_back(n);
          return;
        }
      }
      for (const auto &inp : n->inputs) {
        if (inp.node == replaced) {
          nodes_to_change.push_back(n);
          return;
        }
      }
    });

    // Change references to the `replaced` Node within the `nodes_to_change` list to be
    // references to the equivalent `src` Node.
    for (auto &n : nodes_to_change) {
      for (auto &dep : n->control_deps) {
        if (dep == replaced) {
          dep = src;
        }
      }
      for (auto &inp : n->inputs) {
        if (inp.node == replaced) {
          inp.node = src;
        }
      }
    }

    // Add `replaced` Node control dependencies to those of the `src` Node.
    for (const auto &n : replaced->control_deps) {
      src->control_deps.push_back(n);
    }

    // Change graph outputs driven by the `replaced` Node to now point to the `src` Node.
    for (auto& out : g->outputs) {
      if (out.node == replaced) {
        out.node = src;
      }
    }
  }
  // Check for duplicates in outputs and
  // insert Copy nodes as appropriate
  const Op* copy_op = Op::Get("_copy");
  nnvm::NodeEntryMap<size_t> unique_outputs;
  for (auto & output : g->outputs) {
    auto kv = unique_outputs.find(output);
    if (kv == unique_outputs.end()) {
      unique_outputs.emplace(output, 0);
    } else {
      ObjectPtr copy_node = Node::Create();
      std::ostringstream os;
      os << kv->first.node->attrs.name << "_" << kv->second << "_copy";
      kv->second++;
      copy_node->attrs.op = copy_op;
      copy_node->attrs.name = os.str();
      copy_node->inputs.emplace_back(kv->first);
      output = nnvm::NodeEntry{copy_node, 0, 0};
    }
  }
}

}  // namespace

/*!
 * \brief Simplify a graph by iteratively eliminating Nodes with identical inputs and function.
 */
nnvm::Graph EliminateCommonExpr(nnvm::Graph&& g) {
  using nnvm::ObjectPtr;
  bool keep_running = true;
  while (keep_running) {
    const auto& common_nodes = GetCommonNodes(g);
    if (common_nodes.empty()) {
      keep_running = false;
    } else {
      EliminateCommonNodes(&g, common_nodes);
    }
  }
  return std::move(g);
}

}  // namespace exec
}  // namespace mxnet
<commit_msg>Allow eliminating common subexpressions when temp space is used (#19486)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * Copyright (c) 2019 by Contributors
 * \file eliminate_common_expr.cc
 * \brief Eliminate common expressions in the graph
 * \author Przemyslaw Tredak
 */

#include <mxnet/base.h>
#include <mxnet/op_attr_types.h>

#include <vector>
#include <map>
#include <utility>
#include <sstream>

namespace mxnet {
namespace exec {

namespace {

using nnvm::Node;
using nnvm::ObjectPtr;
using nnvm::Graph;
using nnvm::IndexedGraph;

// NodeInput holds the sufficient subset of NodeEntry fields for Node-input equality tests
using NodeInput = std::pair<const Node*, uint32_t>;

/*!
 * \brief Convert a Node's input vector of `NodeEntry` to a vector of the simpler `NodeInput`
 */
std::vector<NodeInput> ConvertInputs(const std::vector<nnvm::NodeEntry>& inputs) {
  std::vector<NodeInput> ret;
  ret.reserve(inputs.size());
  for (const auto& entry : inputs) {
    ret.emplace_back(entry.node.get(), entry.index);
  }
  return ret;
}

/*!
 * \brief Determine if two Nodes have equal function such that one Node can be eliminated.
 */
bool NodeEqual(const Node* n, const Node* m) {
  if (n->is_variable() || m->is_variable()) return false;
  if (n->op() != m->op()) return false;
  // Nodes with different attributes are considered not identical,
  // though this may reject Node pairs that are in fact functionally the same.
  if (n->attrs.dict != m->attrs.dict) return false;

  // Ops that mutate inputs cannot be optimized out
  static auto& fmutate_inputs = Op::GetAttr<nnvm::FMutateInputs>("FMutateInputs");
  if (fmutate_inputs.get(n->op(), nullptr) != nullptr) return false;

  // Stateful ops cannot be be equal to each other
  static auto& fstateful = Op::GetAttr<FCreateOpState>("FCreateOpState");
  if (fstateful.get(n->op(), nullptr) != nullptr)
    return false;

  // Check to see if the user has explicitly set THasDeterministicOutput to override the
  // subsequent determination of Node equality based on resource use.
  static auto& deterministic_output =
      Op::GetAttr<THasDeterministicOutput>("THasDeterministicOutput");
  if (deterministic_output.contains(n->op()))
    return deterministic_output[n->op()];

  // Ops that require resource could ask for
  // random resource, so need to be explicitly marked
  // to be eligible
  static auto& resource_request = Op::GetAttr<FResourceRequest>("FResourceRequest");
  static auto& resource_request_ex = Op::GetAttr<FResourceRequestEx>("FResourceRequestEx");
  const auto fresource_request = resource_request.get(n->op(), nullptr);
  if (fresource_request != nullptr) {
    const auto& requests = fresource_request(n->attrs);
    for (const auto& req : requests) {
      if (req.type != ResourceRequest::kTempSpace) {
        return false;
      }
    }
  }
  if (resource_request_ex.get(n->op(), nullptr) != nullptr) return false;

  return true;
}

// Graph traversal to create a list of pairs of identical-function nodes that can be combined.
std::vector<std::pair<ObjectPtr, ObjectPtr> > GetCommonNodes(const Graph& g) {
  std::vector<std::pair<ObjectPtr, ObjectPtr> > ret;
  // A map between a vector of inputs and those nodes that have those inputs
  std::map<std::vector<NodeInput>, std::vector<const ObjectPtr*> > grouped_nodes;
  // Traverse the graph and group the nodes by their vector of inputs
  nnvm::DFSVisit(g.outputs, [&grouped_nodes](const ObjectPtr& n) {
    if (n->inputs.size() != 0) {
      grouped_nodes[ConvertInputs(n->inputs)].push_back(&n);
    }
  });
  // Now check for identical node ops within the node groups (having identical inputs)
  for (const auto& pair : grouped_nodes) {
    auto &node_group = pair.second;  // Group of nodes that share the same vector of inputs
    if (node_group.size() > 1) {
      std::unordered_set<size_t> visited;
      for (size_t i = 0; i < node_group.size(); ++i) {
        if (visited.count(i)) continue;
        for (size_t j = i + 1; j < node_group.size(); ++j) {
          // If the two Nodes have equal function, then one Node (called the 'replaced') can
          // be eliminated in favor of the other Node (the 'src').
          if (NodeEqual(node_group[i]->get(), node_group[j]->get())) {
            visited.insert(j);
            ObjectPtr src = *node_group[i];
            ObjectPtr replaced = *node_group[j];
            ret.emplace_back(src, replaced);
          }
        }
      }
    }
  }
  return ret;
}

/*!
 * \brief Do a single pass of Node elimination given pairs of identical Nodes.
 */
void EliminateCommonNodes(Graph* g,
                          const std::vector<std::pair<ObjectPtr, ObjectPtr> >& common_nodes) {
  for (const auto &p : common_nodes) {
    std::vector <ObjectPtr> nodes_to_change;
    const ObjectPtr &src = p.first;
    const ObjectPtr &replaced = p.second;
    // Create a `nodes_to_change` list containing the Nodes that refer to the `replaced` Node
    // that is targeted for elimination.
    DFSVisit(g->outputs, [replaced, &nodes_to_change](const ObjectPtr &n) {
      for (const auto &dep : n->control_deps) {
        if (dep == replaced) {
          nodes_to_change.push_back(n);
          return;
        }
      }
      for (const auto &inp : n->inputs) {
        if (inp.node == replaced) {
          nodes_to_change.push_back(n);
          return;
        }
      }
    });

    // Change references to the `replaced` Node within the `nodes_to_change` list to be
    // references to the equivalent `src` Node.
    for (auto &n : nodes_to_change) {
      for (auto &dep : n->control_deps) {
        if (dep == replaced) {
          dep = src;
        }
      }
      for (auto &inp : n->inputs) {
        if (inp.node == replaced) {
          inp.node = src;
        }
      }
    }

    // Add `replaced` Node control dependencies to those of the `src` Node.
    for (const auto &n : replaced->control_deps) {
      src->control_deps.push_back(n);
    }

    // Change graph outputs driven by the `replaced` Node to now point to the `src` Node.
    for (auto& out : g->outputs) {
      if (out.node == replaced) {
        out.node = src;
      }
    }
  }
  // Check for duplicates in outputs and
  // insert Copy nodes as appropriate
  const Op* copy_op = Op::Get("_copy");
  nnvm::NodeEntryMap<size_t> unique_outputs;
  for (auto & output : g->outputs) {
    auto kv = unique_outputs.find(output);
    if (kv == unique_outputs.end()) {
      unique_outputs.emplace(output, 0);
    } else {
      ObjectPtr copy_node = Node::Create();
      std::ostringstream os;
      os << kv->first.node->attrs.name << "_" << kv->second << "_copy";
      kv->second++;
      copy_node->attrs.op = copy_op;
      copy_node->attrs.name = os.str();
      copy_node->inputs.emplace_back(kv->first);
      output = nnvm::NodeEntry{copy_node, 0, 0};
    }
  }
}

}  // namespace

/*!
 * \brief Simplify a graph by iteratively eliminating Nodes with identical inputs and function.
 */
nnvm::Graph EliminateCommonExpr(nnvm::Graph&& g) {
  using nnvm::ObjectPtr;
  bool keep_running = true;
  while (keep_running) {
    const auto& common_nodes = GetCommonNodes(g);
    if (common_nodes.empty()) {
      keep_running = false;
    } else {
      EliminateCommonNodes(&g, common_nodes);
    }
  }
  return std::move(g);
}

}  // namespace exec
}  // namespace mxnet
<|endoftext|>


<commit_before>/*************************************************************************/
/*  config_file.cpp                                                      */
/*************************************************************************/
/*                       This file is part of:                           */
/*                           GODOT ENGINE                                */
/*                      https://godotengine.org                          */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
/*                                                                       */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the       */
/* "Software"), to deal in the Software without restriction, including   */
/* without limitation the rights to use, copy, modify, merge, publish,   */
/* distribute, sublicense, and/or sell copies of the Software, and to    */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions:                                             */
/*                                                                       */
/* The above copyright notice and this permission notice shall be        */
/* included in all copies or substantial portions of the Software.       */
/*                                                                       */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
/*************************************************************************/

#include "config_file.h"

#include "core/io/file_access_encrypted.h"
#include "core/os/keyboard.h"
#include "core/variant/variant_parser.h"

PackedStringArray ConfigFile::_get_sections() const {
	List<String> s;
	get_sections(&s);
	PackedStringArray arr;
	arr.resize(s.size());
	int idx = 0;
	for (const String &E : s) {
		arr.set(idx++, E);
	}

	return arr;
}

PackedStringArray ConfigFile::_get_section_keys(const String &p_section) const {
	List<String> s;
	get_section_keys(p_section, &s);
	PackedStringArray arr;
	arr.resize(s.size());
	int idx = 0;
	for (const String &E : s) {
		arr.set(idx++, E);
	}

	return arr;
}

void ConfigFile::set_value(const String &p_section, const String &p_key, const Variant &p_value) {
	if (p_value.get_type() == Variant::NIL) {
		//erase
		if (!values.has(p_section)) {
			return; // ?
		}
		values[p_section].erase(p_key);
		if (values[p_section].is_empty()) {
			values.erase(p_section);
		}

	} else {
		if (!values.has(p_section)) {
			values[p_section] = HashMap<String, Variant>();
		}

		values[p_section][p_key] = p_value;
	}
}

Variant ConfigFile::get_value(const String &p_section, const String &p_key, Variant p_default) const {
	if (!values.has(p_section) || !values[p_section].has(p_key)) {
		ERR_FAIL_COND_V_MSG(p_default.get_type() == Variant::NIL, Variant(),
				vformat("Couldn't find the given section \"%s\" and key \"%s\", and no default was given.", p_section, p_key));
		return p_default;
	}

	return values[p_section][p_key];
}

bool ConfigFile::has_section(const String &p_section) const {
	return values.has(p_section);
}

bool ConfigFile::has_section_key(const String &p_section, const String &p_key) const {
	if (!values.has(p_section)) {
		return false;
	}
	return values[p_section].has(p_key);
}

void ConfigFile::get_sections(List<String> *r_sections) const {
	for (const KeyValue<String, HashMap<String, Variant>> &E : values) {
		r_sections->push_back(E.key);
	}
}

void ConfigFile::get_section_keys(const String &p_section, List<String> *r_keys) const {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot get keys from nonexistent section \"%s\".", p_section));

	for (const KeyValue<String, Variant> &E : values[p_section]) {
		r_keys->push_back(E.key);
	}
}

void ConfigFile::erase_section(const String &p_section) {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot erase nonexistent section \"%s\".", p_section));
	values.erase(p_section);
}

void ConfigFile::erase_section_key(const String &p_section, const String &p_key) {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot erase key \"%s\" from nonexistent section \"%s\".", p_key, p_section));
	ERR_FAIL_COND_MSG(!values[p_section].has(p_key), vformat("Cannot erase nonexistent key \"%s\" from section \"%s\".", p_key, p_section));

	values[p_section].erase(p_key);
}

Error ConfigFile::save(const String &p_path) {
	Error err;
	Ref<FileAccess> file = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	return _internal_save(file);
}

Error ConfigFile::save_encrypted(const String &p_path, const Vector<uint8_t> &p_key) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse(f, p_key, FileAccessEncrypted::MODE_WRITE_AES256);
	if (err) {
		return err;
	}
	return _internal_save(fae);
}

Error ConfigFile::save_encrypted_pass(const String &p_path, const String &p_pass) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse_password(f, p_pass, FileAccessEncrypted::MODE_WRITE_AES256);
	if (err) {
		return err;
	}

	return _internal_save(fae);
}

Error ConfigFile::_internal_save(Ref<FileAccess> file) {
	bool first = true;
	for (const KeyValue<String, HashMap<String, Variant>> &E : values) {
		if (first) {
			first = false;
		} else {
			file->store_string("\n");
		}
		if (!E.key.is_empty()) {
			file->store_string("[" + E.key + "]\n\n");
		}

		for (const KeyValue<String, Variant> &F : E.value) {
			String vstr;
			VariantWriter::write_to_string(F.value, vstr);
			file->store_string(F.key.property_name_encode() + "=" + vstr + "\n");
		}
	}

	return OK;
}

Error ConfigFile::load(const String &p_path) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (f.is_null()) {
		return err;
	}

	return _internal_load(p_path, f);
}

Error ConfigFile::load_encrypted(const String &p_path, const Vector<uint8_t> &p_key) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse(f, p_key, FileAccessEncrypted::MODE_READ);
	if (err) {
		return err;
	}
	return _internal_load(p_path, fae);
}

Error ConfigFile::load_encrypted_pass(const String &p_path, const String &p_pass) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse_password(f, p_pass, FileAccessEncrypted::MODE_READ);
	if (err) {
		return err;
	}

	return _internal_load(p_path, fae);
}

Error ConfigFile::_internal_load(const String &p_path, Ref<FileAccess> f) {
	VariantParser::StreamFile stream;
	stream.f = f;

	Error err = _parse(p_path, &stream);

	return err;
}

Error ConfigFile::parse(const String &p_data) {
	VariantParser::StreamString stream;
	stream.s = p_data;
	return _parse("<string>", &stream);
}

Error ConfigFile::_parse(const String &p_path, VariantParser::Stream *p_stream) {
	String assign;
	Variant value;
	VariantParser::Tag next_tag;

	int lines = 0;
	String error_text;

	String section;

	while (true) {
		assign = Variant();
		next_tag.fields.clear();
		next_tag.name = String();

		Error err = VariantParser::parse_tag_assign_eof(p_stream, lines, error_text, next_tag, assign, value, nullptr, true);
		if (err == ERR_FILE_EOF) {
			return OK;
		} else if (err != OK) {
			ERR_PRINT(vformat("ConfigFile parse error at %s:%d: %s.", p_path, lines, error_text));
			return err;
		}

		if (!assign.is_empty()) {
			set_value(section, assign, value);
		} else if (!next_tag.name.is_empty()) {
			section = next_tag.name;
		}
	}

	return OK;
}

void ConfigFile::clear() {
	values.clear();
}
void ConfigFile::_bind_methods() {
	ClassDB::bind_method(D_METHOD("set_value", "section", "key", "value"), &ConfigFile::set_value);
	ClassDB::bind_method(D_METHOD("get_value", "section", "key", "default"), &ConfigFile::get_value, DEFVAL(Variant()));

	ClassDB::bind_method(D_METHOD("has_section", "section"), &ConfigFile::has_section);
	ClassDB::bind_method(D_METHOD("has_section_key", "section", "key"), &ConfigFile::has_section_key);

	ClassDB::bind_method(D_METHOD("get_sections"), &ConfigFile::_get_sections);
	ClassDB::bind_method(D_METHOD("get_section_keys", "section"), &ConfigFile::_get_section_keys);

	ClassDB::bind_method(D_METHOD("erase_section", "section"), &ConfigFile::erase_section);
	ClassDB::bind_method(D_METHOD("erase_section_key", "section", "key"), &ConfigFile::erase_section_key);

	ClassDB::bind_method(D_METHOD("load", "path"), &ConfigFile::load);
	ClassDB::bind_method(D_METHOD("parse", "data"), &ConfigFile::parse);
	ClassDB::bind_method(D_METHOD("save", "path"), &ConfigFile::save);

	BIND_METHOD_ERR_RETURN_DOC("load", ERR_FILE_CANT_OPEN);

	ClassDB::bind_method(D_METHOD("load_encrypted", "path", "key"), &ConfigFile::load_encrypted);
	ClassDB::bind_method(D_METHOD("load_encrypted_pass", "path", "password"), &ConfigFile::load_encrypted_pass);

	ClassDB::bind_method(D_METHOD("save_encrypted", "path", "key"), &ConfigFile::save_encrypted);
	ClassDB::bind_method(D_METHOD("save_encrypted_pass", "path", "password"), &ConfigFile::save_encrypted_pass);

	ClassDB::bind_method(D_METHOD("clear"), &ConfigFile::clear);
}
<commit_msg>Fix saving section-less keys in `ConfigFile`<commit_after>/*************************************************************************/
/*  config_file.cpp                                                      */
/*************************************************************************/
/*                       This file is part of:                           */
/*                           GODOT ENGINE                                */
/*                      https://godotengine.org                          */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
/*                                                                       */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the       */
/* "Software"), to deal in the Software without restriction, including   */
/* without limitation the rights to use, copy, modify, merge, publish,   */
/* distribute, sublicense, and/or sell copies of the Software, and to    */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions:                                             */
/*                                                                       */
/* The above copyright notice and this permission notice shall be        */
/* included in all copies or substantial portions of the Software.       */
/*                                                                       */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
/*************************************************************************/

#include "config_file.h"

#include "core/io/file_access_encrypted.h"
#include "core/os/keyboard.h"
#include "core/variant/variant_parser.h"

PackedStringArray ConfigFile::_get_sections() const {
	List<String> s;
	get_sections(&s);
	PackedStringArray arr;
	arr.resize(s.size());
	int idx = 0;
	for (const String &E : s) {
		arr.set(idx++, E);
	}

	return arr;
}

PackedStringArray ConfigFile::_get_section_keys(const String &p_section) const {
	List<String> s;
	get_section_keys(p_section, &s);
	PackedStringArray arr;
	arr.resize(s.size());
	int idx = 0;
	for (const String &E : s) {
		arr.set(idx++, E);
	}

	return arr;
}

void ConfigFile::set_value(const String &p_section, const String &p_key, const Variant &p_value) {
	if (p_value.get_type() == Variant::NIL) { // Erase key.
		if (!values.has(p_section)) {
			return;
		}

		values[p_section].erase(p_key);
		if (values[p_section].is_empty()) {
			values.erase(p_section);
		}
	} else {
		if (!values.has(p_section)) {
			// Insert section-less keys at the beginning.
			values.insert(p_section, HashMap<String, Variant>(), p_section.is_empty());
		}

		values[p_section][p_key] = p_value;
	}
}

Variant ConfigFile::get_value(const String &p_section, const String &p_key, Variant p_default) const {
	if (!values.has(p_section) || !values[p_section].has(p_key)) {
		ERR_FAIL_COND_V_MSG(p_default.get_type() == Variant::NIL, Variant(),
				vformat("Couldn't find the given section \"%s\" and key \"%s\", and no default was given.", p_section, p_key));
		return p_default;
	}

	return values[p_section][p_key];
}

bool ConfigFile::has_section(const String &p_section) const {
	return values.has(p_section);
}

bool ConfigFile::has_section_key(const String &p_section, const String &p_key) const {
	if (!values.has(p_section)) {
		return false;
	}
	return values[p_section].has(p_key);
}

void ConfigFile::get_sections(List<String> *r_sections) const {
	for (const KeyValue<String, HashMap<String, Variant>> &E : values) {
		r_sections->push_back(E.key);
	}
}

void ConfigFile::get_section_keys(const String &p_section, List<String> *r_keys) const {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot get keys from nonexistent section \"%s\".", p_section));

	for (const KeyValue<String, Variant> &E : values[p_section]) {
		r_keys->push_back(E.key);
	}
}

void ConfigFile::erase_section(const String &p_section) {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot erase nonexistent section \"%s\".", p_section));
	values.erase(p_section);
}

void ConfigFile::erase_section_key(const String &p_section, const String &p_key) {
	ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot erase key \"%s\" from nonexistent section \"%s\".", p_key, p_section));
	ERR_FAIL_COND_MSG(!values[p_section].has(p_key), vformat("Cannot erase nonexistent key \"%s\" from section \"%s\".", p_key, p_section));

	values[p_section].erase(p_key);
	if (values[p_section].is_empty()) {
		values.erase(p_section);
	}
}

Error ConfigFile::save(const String &p_path) {
	Error err;
	Ref<FileAccess> file = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	return _internal_save(file);
}

Error ConfigFile::save_encrypted(const String &p_path, const Vector<uint8_t> &p_key) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse(f, p_key, FileAccessEncrypted::MODE_WRITE_AES256);
	if (err) {
		return err;
	}
	return _internal_save(fae);
}

Error ConfigFile::save_encrypted_pass(const String &p_path, const String &p_pass) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::WRITE, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse_password(f, p_pass, FileAccessEncrypted::MODE_WRITE_AES256);
	if (err) {
		return err;
	}

	return _internal_save(fae);
}

Error ConfigFile::_internal_save(Ref<FileAccess> file) {
	bool first = true;
	for (const KeyValue<String, HashMap<String, Variant>> &E : values) {
		if (first) {
			first = false;
		} else {
			file->store_string("\n");
		}
		if (!E.key.is_empty()) {
			file->store_string("[" + E.key + "]\n\n");
		}

		for (const KeyValue<String, Variant> &F : E.value) {
			String vstr;
			VariantWriter::write_to_string(F.value, vstr);
			file->store_string(F.key.property_name_encode() + "=" + vstr + "\n");
		}
	}

	return OK;
}

Error ConfigFile::load(const String &p_path) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (f.is_null()) {
		return err;
	}

	return _internal_load(p_path, f);
}

Error ConfigFile::load_encrypted(const String &p_path, const Vector<uint8_t> &p_key) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse(f, p_key, FileAccessEncrypted::MODE_READ);
	if (err) {
		return err;
	}
	return _internal_load(p_path, fae);
}

Error ConfigFile::load_encrypted_pass(const String &p_path, const String &p_pass) {
	Error err;
	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

	if (err) {
		return err;
	}

	Ref<FileAccessEncrypted> fae;
	fae.instantiate();
	err = fae->open_and_parse_password(f, p_pass, FileAccessEncrypted::MODE_READ);
	if (err) {
		return err;
	}

	return _internal_load(p_path, fae);
}

Error ConfigFile::_internal_load(const String &p_path, Ref<FileAccess> f) {
	VariantParser::StreamFile stream;
	stream.f = f;

	Error err = _parse(p_path, &stream);

	return err;
}

Error ConfigFile::parse(const String &p_data) {
	VariantParser::StreamString stream;
	stream.s = p_data;
	return _parse("<string>", &stream);
}

Error ConfigFile::_parse(const String &p_path, VariantParser::Stream *p_stream) {
	String assign;
	Variant value;
	VariantParser::Tag next_tag;

	int lines = 0;
	String error_text;

	String section;

	while (true) {
		assign = Variant();
		next_tag.fields.clear();
		next_tag.name = String();

		Error err = VariantParser::parse_tag_assign_eof(p_stream, lines, error_text, next_tag, assign, value, nullptr, true);
		if (err == ERR_FILE_EOF) {
			return OK;
		} else if (err != OK) {
			ERR_PRINT(vformat("ConfigFile parse error at %s:%d: %s.", p_path, lines, error_text));
			return err;
		}

		if (!assign.is_empty()) {
			set_value(section, assign, value);
		} else if (!next_tag.name.is_empty()) {
			section = next_tag.name;
		}
	}

	return OK;
}

void ConfigFile::clear() {
	values.clear();
}
void ConfigFile::_bind_methods() {
	ClassDB::bind_method(D_METHOD("set_value", "section", "key", "value"), &ConfigFile::set_value);
	ClassDB::bind_method(D_METHOD("get_value", "section", "key", "default"), &ConfigFile::get_value, DEFVAL(Variant()));

	ClassDB::bind_method(D_METHOD("has_section", "section"), &ConfigFile::has_section);
	ClassDB::bind_method(D_METHOD("has_section_key", "section", "key"), &ConfigFile::has_section_key);

	ClassDB::bind_method(D_METHOD("get_sections"), &ConfigFile::_get_sections);
	ClassDB::bind_method(D_METHOD("get_section_keys", "section"), &ConfigFile::_get_section_keys);

	ClassDB::bind_method(D_METHOD("erase_section", "section"), &ConfigFile::erase_section);
	ClassDB::bind_method(D_METHOD("erase_section_key", "section", "key"), &ConfigFile::erase_section_key);

	ClassDB::bind_method(D_METHOD("load", "path"), &ConfigFile::load);
	ClassDB::bind_method(D_METHOD("parse", "data"), &ConfigFile::parse);
	ClassDB::bind_method(D_METHOD("save", "path"), &ConfigFile::save);

	BIND_METHOD_ERR_RETURN_DOC("load", ERR_FILE_CANT_OPEN);

	ClassDB::bind_method(D_METHOD("load_encrypted", "path", "key"), &ConfigFile::load_encrypted);
	ClassDB::bind_method(D_METHOD("load_encrypted_pass", "path", "password"), &ConfigFile::load_encrypted_pass);

	ClassDB::bind_method(D_METHOD("save_encrypted", "path", "key"), &ConfigFile::save_encrypted);
	ClassDB::bind_method(D_METHOD("save_encrypted_pass", "path", "password"), &ConfigFile::save_encrypted_pass);

	ClassDB::bind_method(D_METHOD("clear"), &ConfigFile::clear);
}
<|endoftext|>


<commit_before>/*
 * File:   SDFLoader.cpp
 * Author: Benedikt Vogler
 *
 * Created on 8. Juli 2014, 18:52
 */

#include <algorithm>
#include "SDFloader.hpp"
#include "Camera.hpp"
#include "Box.hpp"
#include "Sphere.hpp"
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include "Material.hpp"
#include "LightPoint.hpp"
#include "Composite.hpp"

using namespace std;

/**
 * Loads a scene file and reacts to the commands in it
 * @param scenefile the string to the file
 * @return
 */
Scene SDFLoader::load(std::string const& scenefile) {
    cout << "Loading file: " << scenefile << endl;
    Scene scene = Scene();

    std::map<std::string, Material> mMap;
    std::map<std::string, LightPoint> lMap;
    std::map<std::string, RenderObject*> roMap;

    string line;
    ifstream file(scenefile);
    stringstream ss;
    //file.open(scenefile, ios::in);
    if (file.is_open()) {
        while (getline (file,line)){//get line
            ss = stringstream(line);//fill the line into stringstream
            string firstWord;
            ss >> firstWord;
            //is first string define?
            if (firstWord=="define"){
                cout << "defininig: ";

                ss >> firstWord;
                if (firstWord=="material"){
                    cout << "a material: "<<endl;
                    //extract name
                    string name;
                    ss>>name;

                    //extract color
                    float red, green, blue;
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color ca(red, green, blue);
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color cd(red, green, blue);
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color cs(red, green, blue);
                    float m;
                    ss >> m;
                    Material mat(ca, cd, cs,m, name);
                    cout << "Material specs: "<<endl<<mat;
                    mMap[name]=mat;
                } else if (firstWord=="camera"){
                    string cameraname;
                    ss >> cameraname;
                    int fovX;
                    ss >> fovX;
                    scene.camera = Camera(cameraname,fovX);
                    cout << "camera: "<<cameraname<<"("<<fovX<<")"<<endl;
                } else if (firstWord=="light"){
                    string type;
                    ss>>type;

                    if (type=="diffuse") {
                        string name;
                        ss >> name;

                        glm::vec3 pos;
                        ss >> pos.x;
                        ss >> pos.y;
                        ss >> pos.z;

                        float red, green, blue;
                        ss >> red;
                        ss >> green;
                        ss >> blue;
                        Color diff(red, green, blue);

                        LightPoint light = LightPoint(name, pos, diff);
                        cout << "light point: "<<name<<"("<<pos.x<<","<<pos.y<<","<<pos.z<<","<<diff<<")"<<endl;
                        lMap[name] = light;
                    }else if (type=="ambient") {
                        string lightname;
                        ss >> lightname;//name get's ignored

                        float red, green, blue;
                        ss >> red;
                        ss >> green;
                        ss >> blue;
                        Color amb(red, green, blue);

                        scene.amb = amb;
                        cout << "ambient light "<<amb<<endl;
                    } else {
                        cout << "type not supported yet."<<endl;
                    }
                } else if (firstWord=="shape"){
                    string classname;
                    ss >> classname;
                    cout << "Shape \""<< classname << "\"."<<endl;
                    transform(classname.begin(), classname.end(),classname.begin(), ::toupper);

                    string name;
                    ss >> name;

                    RenderObject* rObject = nullptr;
                    if (classname=="BOX"){
                        int edge1x, edge1y, edge1z;
                        ss>> edge1x;
                        ss>> edge1y;
                        ss>> edge1z;

                        int edge2x, edge2y, edge2z;
                        ss>> edge2x;
                        ss>> edge2y;
                        ss>> edge2z;

                        string materialName;
                        ss>>materialName;
                        Material material = mMap[materialName];

                        rObject = new Box(
                                name,
                                glm::vec3(edge1x, edge1y, edge1z),
                                glm::vec3(edge2x, edge2y, edge2z),
                                material
                                );
                    } else if (classname=="SPHERE") {
                        int posX, posY, posZ;
                        ss>> posX;
                        ss>> posY;
                        ss>> posZ;
                        float radius;
                        ss>>radius;

                        string materialString;
                        ss>>materialString;

                        rObject = new Sphere(
                            name,
                            glm::vec3(posX, posY, posZ),
                            radius,
                            mMap[materialString]
                        );
                        cout << "Sphere \""<< name << "\" aus Material "<<materialString<<" mit Radius: "<<radius<<"@("<<posX<<","<<posY<<","<<posZ<<")"<<endl;
                    } else if(classname=="COMPOSITE") {
                        rObject = new Composite(name);
                        cout << "Composite \""<< name << "\" (" ;
                        string objectString;
                        while (!ss.eof()){
                            ss>>objectString;
                            auto linkedObject = roMap.find(objectString);
                            if (linkedObject == roMap.end()){
                                cout << "Error: "<<objectString <<" not found!";
                            } else {
                                ((Composite*)rObject)->add_child(linkedObject->second);
                                cout<<", "<<objectString;
                            }
                        }
                        cout<<")"<<endl;
                    } else cout << "ERROR: Shape \""<< classname << "\" not defined."<<endl;

                    if (rObject != nullptr)
                        roMap[name] = rObject;
                } else
                    cout << "object to define not implemented:"<<ss.str() <<endl;
            } else if (firstWord=="render"){
                ss >> scene.camname;
                ss >> scene.outputFile;
                ss >> scene.resX;
                ss >> scene.resY;
                ss >> scene.antialiase;

                //set default if not set
                if (scene.resX<=0) scene.resX=480;
                if (scene.resY<=0) scene.resY=320;
                cout << "Scene should be rendered from "<< scene.camname << " at resolution "<<scene.resX<<"x"<< scene.resY<<"with "<< scene.antialiase<<"x SSAA to "<<scene.outputFile<<endl;
            } else if (firstWord=="#" || firstWord.substr(0,1)=="#"){
                cout << line << endl;//just print comment lines

            } else if (firstWord == "transform"){
                string name, transform;
                double X, Y, Z;
                RenderObject* object;

                ss >> name;
                ss >> transform;

                auto linkedObject = roMap.find(name);
                if (linkedObject == roMap.end()){//check if object can be found
                    cout << "Error: " << name << " not found!";
                } else {
                    object = linkedObject->second;
                    if (transform == "scale") {
                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->scale(coords);
                    } else if (transform == "rotate") {

                        double angle;
                        ss >> angle;

                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->rotate(angle,coords);

                    } else if (transform == "translate"){
                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->translate(coords);

                    } else {
                            cout << "Unknown transformation" << endl;
                    }
                }
            } else cout << "???:"<<line <<endl;//print unknown lines
        }
        file.close();
    }else cout << "Unable to open file";

    //save collected data in scene
    scene.materials = mMap;
    scene.renderObjects = roMap;
    scene.lights = lMap;
    return scene;
}<commit_msg>improved logging<commit_after>/*
 * File:   SDFLoader.cpp
 * Author: Benedikt Vogler
 *
 * Created on 8. Juli 2014, 18:52
 */

#include <algorithm>
#include "SDFloader.hpp"
#include "Camera.hpp"
#include "Box.hpp"
#include "Sphere.hpp"
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include "Material.hpp"
#include "LightPoint.hpp"
#include "Composite.hpp"

using namespace std;

/**
 * Loads a scene file and reacts to the commands in it
 * @param scenefile the string to the file
 * @return
 */
Scene SDFLoader::load(std::string const& scenefile) {
    cout << "Loading file: " << scenefile << endl;
    Scene scene = Scene();

    std::map<std::string, Material> mMap;
    std::map<std::string, LightPoint> lMap;
    std::map<std::string, RenderObject*> roMap;

    string line;
    ifstream file(scenefile);
    stringstream ss;
    //file.open(scenefile, ios::in);
    if (file.is_open()) {
        while (getline (file,line)){//get line
            ss = stringstream(line);//fill the line into stringstream
            string firstWord;
            ss >> firstWord;
            //is first string define?
            if (firstWord=="define"){
                cout << "defininig: ";

                ss >> firstWord;
                if (firstWord=="material"){
                    cout << "a material: "<<endl;
                    //extract name
                    string name;
                    ss>>name;

                    //extract color
                    float red, green, blue;
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color ca(red, green, blue);
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color cd(red, green, blue);
                    ss >> red;
                    ss >> green;
                    ss >> blue;
                    Color cs(red, green, blue);
                    float m;
                    ss >> m;
                    Material mat(ca, cd, cs,m, name);
                    cout << "Material specs: "<<endl<<mat;
                    mMap[name]=mat;
                } else if (firstWord=="camera"){
                    string cameraname;
                    ss >> cameraname;
                    int fovX;
                    ss >> fovX;
                    scene.camera = Camera(cameraname,fovX);
                    cout << "camera: "<<cameraname<<"("<<fovX<<")"<<endl;
                } else if (firstWord=="light"){
                    string type;
                    ss>>type;

                    if (type=="diffuse") {
                        string name;
                        ss >> name;

                        glm::vec3 pos;
                        ss >> pos.x;
                        ss >> pos.y;
                        ss >> pos.z;

                        float red, green, blue;
                        ss >> red;
                        ss >> green;
                        ss >> blue;
                        Color diff(red, green, blue);

                        LightPoint light = LightPoint(name, pos, diff);
                        cout << "light point: "<<name<<"("<<pos.x<<","<<pos.y<<","<<pos.z<<","<<diff<<")"<<endl;
                        lMap[name] = light;
                    }else if (type=="ambient") {
                        string lightname;
                        ss >> lightname;//name get's ignored

                        float red, green, blue;
                        ss >> red;
                        ss >> green;
                        ss >> blue;
                        Color amb(red, green, blue);

                        scene.amb = amb;
                        cout << "ambient light "<<amb<<endl;
                    } else {
                        cout << "type not supported yet."<<endl;
                    }
                } else if (firstWord=="shape"){
                    string classname;
                    ss >> classname;
                    transform(classname.begin(), classname.end(),classname.begin(), ::toupper);

                    string name;
                    ss >> name;

                    RenderObject* rObject = nullptr;
                    if (classname=="BOX"){
                        int edge1x, edge1y, edge1z;
                        ss>> edge1x;
                        ss>> edge1y;
                        ss>> edge1z;

                        int edge2x, edge2y, edge2z;
                        ss>> edge2x;
                        ss>> edge2y;
                        ss>> edge2z;

                        string materialName;
                        ss>>materialName;
                        Material material = mMap[materialName];

                        rObject = new Box(
                                name,
                                glm::vec3(edge1x, edge1y, edge1z),
                                glm::vec3(edge2x, edge2y, edge2z),
                                material
                                );
                    } else if (classname=="SPHERE") {
                        int posX, posY, posZ;
                        ss>> posX;
                        ss>> posY;
                        ss>> posZ;
                        float radius;
                        ss>>radius;

                        string materialString;
                        ss>>materialString;

                        rObject = new Sphere(
                            name,
                            glm::vec3(posX, posY, posZ),
                            radius,
                            mMap[materialString]
                        );
                        cout << "Sphere \""<< name << "\" aus Material "<<materialString<<" mit Radius: "<<radius<<"@("<<posX<<","<<posY<<","<<posZ<<")"<<endl;
                    } else if(classname=="COMPOSITE") {
                        rObject = new Composite(name);
                        cout << "Composite \""<< name << "\" (" ;
                        string objectString;
                        while (!ss.eof()){
                            ss>>objectString;
                            auto linkedObject = roMap.find(objectString);
                            if (linkedObject == roMap.end()){
                                cout << "Error: "<<objectString <<" not found!";
                            } else {
                                ((Composite*)rObject)->add_child(linkedObject->second);
                                cout<<", "<<objectString;
                            }
                        }
                        cout<<")"<<endl;
                    } else cout << "ERROR: Shape \""<< classname << "\" not defined."<<endl;

                    if (rObject != nullptr)
                        roMap[name] = rObject;
                } else
                    cout << "object to define not implemented:"<<ss.str() <<endl;
            } else if (firstWord=="render"){
                ss >> scene.camname;
                ss >> scene.outputFile;
                ss >> scene.resX;
                ss >> scene.resY;
                ss >> scene.antialiase;

                //set default if not set
                if (scene.resX<=0) scene.resX=480;
                if (scene.resY<=0) scene.resY=320;
                cout << "Scene should be rendered from "<< scene.camname << " at resolution "<<scene.resX<<"x"<< scene.resY<<"with "<< scene.antialiase<<"x SSAA to "<<scene.outputFile<<endl;
            } else if (firstWord=="#" || firstWord.substr(0,1)=="#"){
                cout << line << endl;//just print comment lines

            } else if (firstWord == "transform"){
                string name, transform;
                double X, Y, Z;
                RenderObject* object;

                ss >> name;
                ss >> transform;

                auto linkedObject = roMap.find(name);
                if (linkedObject == roMap.end()){//check if object can be found
                    cout << "Error: " << name << " not found!";
                } else {
                    object = linkedObject->second;
                    if (transform == "scale") {
                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->scale(coords);
                    } else if (transform == "rotate") {

                        double angle;
                        ss >> angle;

                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->rotate(angle,coords);

                    } else if (transform == "translate"){
                        ss >> X;
                        ss >> Y;
                        ss >> Z;

                        glm::vec3 coords(X, Y, Z);
                        object->translate(coords);

                    } else {
                            cout << "Unknown transformation" << endl;
                    }
                }
            } else if (firstWord.length()<1){

            } else cout << "???:"<<line <<endl;//print unknown lines
        }
        file.close();
    }else cout << "Unable to open file";

    //save collected data in scene
    scene.materials = mMap;
    scene.renderObjects = roMap;
    scene.lights = lMap;
    return scene;
}<|endoftext|>


<commit_before>// clang-format off

CppKeySet { 10,
	    keyNew (PREFIX "key", KEY_END),
	    keyNew (PREFIX "key/map", KEY_END),
	    keyNew (PREFIX "key/array", KEY_END),
	    keyNew (PREFIX "key/array/#0", KEY_END),
	    keyNew (PREFIX "key/array/#1", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested/#0", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested/#1", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array/#0", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array/key", KEY_END),
	    keyNew (PREFIX "key/array/#4/array/without/parent/#0", KEY_END),
	    keyNew (PREFIX "key/array/#4/array/without/parent/#1", KEY_END),
	    keyNew (PREFIX "key/empty/array", KEY_META, "array", "", KEY_END),
	    KS_END }
<commit_msg>Directory Value: Update test data<commit_after>// clang-format off

CppKeySet { 10,
	    keyNew (PREFIX "key", KEY_END),
	    keyNew (PREFIX "key/map", KEY_END),
	    keyNew (PREFIX "key/array", KEY_META, "array", "#4", KEY_END),
	    keyNew (PREFIX "key/array/#0", KEY_END),
	    keyNew (PREFIX "key/array/#1", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested", KEY_META, "array", "#1", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested/#0", KEY_END),
	    keyNew (PREFIX "key/array/#2/nested/#1", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array/#0", KEY_END),
	    keyNew (PREFIX "key/array/#3/not/an/array/key", KEY_END),
	    keyNew (PREFIX "key/array/#4/no/array/#0", KEY_END),
	    keyNew (PREFIX "key/array/#4/no/array/#1", KEY_END),
	    keyNew (PREFIX "key/empty/array", KEY_META, "array", "", KEY_END),
	    KS_END }
<|endoftext|>


<commit_before>#include "AutoTrace.h"

// ITK
#include "itkAndImageFilter.h"
#include "itkBinaryBallStructuringElement.h"
#include "itkBinaryDilateImageFilter.h"
#include "itkBinaryThresholdImageFilter.h"
#include "itkConnectedThresholdImageFilter.h"
#include "itkImage.h"
#include "itkVectorIndexSelectionCastImageFilter.h"

void Trace(const RGBImageType::Pointer image, const itk::CovariantVector<unsigned char, 3>& color, const itk::Index<2>& seed)
{
  // Create a color epsilon lower than the user supplied color
  int epsilon = 10;
  RGBImageType::PixelType lowerColor;
  RGBImageType::PixelType upperColor;
  for(unsigned int component = 0; component < 3; ++component)
    {
    lowerColor[component] = std::max(static_cast<int>(color[component]) - epsilon, 0); // Truncate negative values
    upperColor[component] = std::min(static_cast<int>(color[component]) + epsilon, 255); // Truncate values above 255
    }
  std::cout << "Lower color: ";
  for(unsigned int i = 0; i < 3; ++i)
    {
    std::cout << static_cast<int>(lowerColor[i]) << " ";
    }
  std::cout << std::endl;

  std::cout << "Upper color: ";
  for(unsigned int i = 0; i < 3; ++i)
    {
    std::cout << static_cast<int>(upperColor[i]) << " ";
    }
  std::cout << std::endl;
  // Threshold all 3 channels and AND the result
  std::vector<ScalarImageType::Pointer> thresholdedChannels(3);
  for(unsigned int channel = 0; channel < 3; ++channel)
  {
    typedef itk::VectorIndexSelectionCastImageFilter<RGBImageType, ScalarImageType> IndexSelectionType;
    IndexSelectionType::Pointer indexSelectionFilter = IndexSelectionType::New();
    indexSelectionFilter->SetIndex(channel);
    indexSelectionFilter->SetInput(image);
    indexSelectionFilter->Update();

    // Use the color of the selection to threshold the image (looking for similar colors)
    typedef itk::BinaryThresholdImageFilter <ScalarImageType, ScalarImageType> BinaryThresholdImageFilterType;
    // Color pixels inside the threshold white. Color pixels outside the threshold black.
    BinaryThresholdImageFilterType::Pointer thresholdFilter = BinaryThresholdImageFilterType::New();
    thresholdFilter->SetInput(indexSelectionFilter->GetOutput());
    thresholdFilter->SetLowerThreshold(lowerColor[channel]);
    thresholdFilter->SetUpperThreshold(upperColor[channel]);
    thresholdFilter->SetInsideValue(255); // White
    thresholdFilter->SetOutsideValue(0); // Black
    thresholdFilter->Update();

    thresholdedChannels[channel] = ScalarImageType::New();
    DeepCopy<ScalarImageType>(thresholdFilter->GetOutput(), thresholdedChannels[channel]);
    }

  // Keep only region where all channels passed the threshold test
  typedef itk::AndImageFilter <ScalarImageType> AndImageFilterType;

  AndImageFilterType::Pointer andFilter1 = AndImageFilterType::New();
  andFilter1->SetInput(0, thresholdedChannels[0]);
  andFilter1->SetInput(1, thresholdedChannels[1]);
  andFilter1->Update();

  AndImageFilterType::Pointer andFilter2 = AndImageFilterType::New();
  andFilter2->SetInput(0, andFilter1->GetOutput());
  andFilter2->SetInput(1, thresholdedChannels[2]);
  andFilter2->Update();

  ScalarImageType::Pointer thresholdedImage = ScalarImageType::New();
  DeepCopy<ScalarImageType>(andFilter2->GetOutput(), thresholdedImage);
  WriteImage<ScalarImageType>(thresholdedImage, "Thresholded.png");

  // Dilate the image.
  typedef itk::BinaryBallStructuringElement<ScalarImageType::PixelType, 2> StructuringElementType;
  StructuringElementType structuringElement;
  unsigned int radius = 5; // How big of holes we want to connect.
  structuringElement.SetRadius(radius);
  structuringElement.CreateStructuringElement();

  typedef itk::BinaryDilateImageFilter <ScalarImageType, ScalarImageType, StructuringElementType> BinaryDilateImageFilterType;
  BinaryDilateImageFilterType::Pointer dilateFilter = BinaryDilateImageFilterType::New();
  dilateFilter->SetInput(thresholdedImage);
  dilateFilter->SetKernel(structuringElement);
  dilateFilter->SetDilateValue(255);
  dilateFilter->Update();

  ScalarImageType::Pointer dilatedImage = ScalarImageType::New();
  DeepCopy<ScalarImageType>(dilateFilter->GetOutput(), dilatedImage);
  WriteImage<ScalarImageType>(dilatedImage, "Dilated.png");

  // Get pixels that are connected to the selected point.
  typedef itk::ConnectedThresholdImageFilter<ScalarImageType, ScalarImageType> ConnectedFilterType;
  ConnectedFilterType::Pointer connectedThreshold = ConnectedFilterType::New();
  connectedThreshold->SetLower(255);
  connectedThreshold->SetUpper(255);
  connectedThreshold->SetReplaceValue(255);
  connectedThreshold->SetSeed(seed);
  connectedThreshold->SetInput(dilatedImage);
  connectedThreshold->Update();

  WriteImage<ScalarImageType>(connectedThreshold->GetOutput(), "connectedThreshold.png");
}
<commit_msg>Skeletonize the image after the dilation of the thresholded image and before the connectivity search.<commit_after>#include "AutoTrace.h"

// ITK
#include "itkAndImageFilter.h"
#include "itkBinaryBallStructuringElement.h"
#include "itkBinaryDilateImageFilter.h"
#include "itkBinaryThinningImageFilter.h"
#include "itkBinaryThresholdImageFilter.h"
#include "itkConnectedThresholdImageFilter.h"
#include "itkImage.h"
#include "itkRescaleIntensityImageFilter.h"
#include "itkVectorIndexSelectionCastImageFilter.h"

void Trace(const RGBImageType::Pointer image, const itk::CovariantVector<unsigned char, 3>& color, const itk::Index<2>& seed)
{
  // Create a color epsilon lower than the user supplied color
  int epsilon = 10;
  RGBImageType::PixelType lowerColor;
  RGBImageType::PixelType upperColor;
  for(unsigned int component = 0; component < 3; ++component)
    {
    lowerColor[component] = std::max(static_cast<int>(color[component]) - epsilon, 0); // Truncate negative values
    upperColor[component] = std::min(static_cast<int>(color[component]) + epsilon, 255); // Truncate values above 255
    }
  std::cout << "Lower color: ";
  for(unsigned int i = 0; i < 3; ++i)
    {
    std::cout << static_cast<int>(lowerColor[i]) << " ";
    }
  std::cout << std::endl;

  std::cout << "Upper color: ";
  for(unsigned int i = 0; i < 3; ++i)
    {
    std::cout << static_cast<int>(upperColor[i]) << " ";
    }
  std::cout << std::endl;
  // Threshold all 3 channels and AND the result
  std::vector<ScalarImageType::Pointer> thresholdedChannels(3);
  for(unsigned int channel = 0; channel < 3; ++channel)
  {
    typedef itk::VectorIndexSelectionCastImageFilter<RGBImageType, ScalarImageType> IndexSelectionType;
    IndexSelectionType::Pointer indexSelectionFilter = IndexSelectionType::New();
    indexSelectionFilter->SetIndex(channel);
    indexSelectionFilter->SetInput(image);
    indexSelectionFilter->Update();

    // Use the color of the selection to threshold the image (looking for similar colors)
    typedef itk::BinaryThresholdImageFilter <ScalarImageType, ScalarImageType> BinaryThresholdImageFilterType;
    // Color pixels inside the threshold white. Color pixels outside the threshold black.
    BinaryThresholdImageFilterType::Pointer thresholdFilter = BinaryThresholdImageFilterType::New();
    thresholdFilter->SetInput(indexSelectionFilter->GetOutput());
    thresholdFilter->SetLowerThreshold(lowerColor[channel]);
    thresholdFilter->SetUpperThreshold(upperColor[channel]);
    thresholdFilter->SetInsideValue(255); // White
    thresholdFilter->SetOutsideValue(0); // Black
    thresholdFilter->Update();

    thresholdedChannels[channel] = ScalarImageType::New();
    DeepCopy<ScalarImageType>(thresholdFilter->GetOutput(), thresholdedChannels[channel]);
    }

  // Keep only region where all channels passed the threshold test
  typedef itk::AndImageFilter <ScalarImageType> AndImageFilterType;

  AndImageFilterType::Pointer andFilter1 = AndImageFilterType::New();
  andFilter1->SetInput(0, thresholdedChannels[0]);
  andFilter1->SetInput(1, thresholdedChannels[1]);
  andFilter1->Update();

  AndImageFilterType::Pointer andFilter2 = AndImageFilterType::New();
  andFilter2->SetInput(0, andFilter1->GetOutput());
  andFilter2->SetInput(1, thresholdedChannels[2]);
  andFilter2->Update();

  ScalarImageType::Pointer thresholdedImage = ScalarImageType::New();
  DeepCopy<ScalarImageType>(andFilter2->GetOutput(), thresholdedImage);
  WriteImage<ScalarImageType>(thresholdedImage, "Thresholded.png");

  // Dilate the image.
  typedef itk::BinaryBallStructuringElement<ScalarImageType::PixelType, 2> StructuringElementType;
  StructuringElementType structuringElement;
  unsigned int radius = 5; // How big of holes we want to connect.
  structuringElement.SetRadius(radius);
  structuringElement.CreateStructuringElement();

  typedef itk::BinaryDilateImageFilter <ScalarImageType, ScalarImageType, StructuringElementType> BinaryDilateImageFilterType;
  BinaryDilateImageFilterType::Pointer dilateFilter = BinaryDilateImageFilterType::New();
  dilateFilter->SetInput(thresholdedImage);
  dilateFilter->SetKernel(structuringElement);
  dilateFilter->SetDilateValue(255);
  dilateFilter->Update();

  ScalarImageType::Pointer dilatedImage = ScalarImageType::New();
  DeepCopy<ScalarImageType>(dilateFilter->GetOutput(), dilatedImage);
  WriteImage<ScalarImageType>(dilatedImage, "Dilated.png");

  // Thin/skeletonize the image
  typedef itk::BinaryThinningImageFilter <ScalarImageType, ScalarImageType> BinaryThinningImageFilterType;
  BinaryThinningImageFilterType::Pointer binaryThinningImageFilter = BinaryThinningImageFilterType::New();
  binaryThinningImageFilter->SetInput(dilatedImage);
  binaryThinningImageFilter->Update();

  // Rescale the output of the thinning filter so that it can be seen (the output is 0 and 1, we want 0 and 255)
  typedef itk::RescaleIntensityImageFilter< ScalarImageType, ScalarImageType > RescaleType;
  RescaleType::Pointer rescaler = RescaleType::New();
  rescaler->SetInput( binaryThinningImageFilter->GetOutput() );
  rescaler->SetOutputMinimum(0);
  rescaler->SetOutputMaximum(255);
  rescaler->Update();

  WriteImage<ScalarImageType>(rescaler->GetOutput(), "Thinned.png");

  // Get pixels that are connected to the selected point.
  typedef itk::ConnectedThresholdImageFilter<ScalarImageType, ScalarImageType> ConnectedFilterType;
  ConnectedFilterType::Pointer connectedThreshold = ConnectedFilterType::New();
  connectedThreshold->SetLower(255);
  connectedThreshold->SetUpper(255);
  connectedThreshold->SetReplaceValue(255);
  connectedThreshold->SetSeed(seed);
  connectedThreshold->SetInput(rescaler->GetOutput());
  connectedThreshold->SetConnectivity(ConnectedFilterType::FullConnectivity); // consider 8 neighbors when deciding connectivity
  connectedThreshold->Update();

  WriteImage<ScalarImageType>(connectedThreshold->GetOutput(), "connectedThreshold.png");
}
<|endoftext|>