text: // Copyright (C) 2015  Davis E. King (davis@dlib.net)
  // License: Boost Software License   See LICENSE.txt for the full license.
  #ifndef DLIB_DNn_LAYERS_H_
  #define DLIB_DNn_LAYERS_H_
  
  #include "layers_abstract.h"
  #include "tensor.h"
  #include "core.h"
  #include <iostream>
  #include <string>
  #include "../rand.h"
  #include "../string.h"
  #include "tensor_tools.h"
  #include "../vectorstream.h"
  #include "utilities.h"
  #include <sstream>
  
  
  namespace dlib
  {
  
  // ----------------------------------------------------------------------------------------
  
      template <
          long _num_filters,
          long _nr,
          long _nc,
          int _stride_y,
          int _stride_x,
          int _padding_y = _stride_y!=1? 0 : _nr/2,
          int _padding_x = _stride_x!=1? 0 : _nc/2
          >
      class con_
      {
      public:
  
          static_assert(_num_filters > 0, "The number of filters must be > 0");
          static_assert(_nr > 0, "The number of rows in a filter must be > 0");
          static_assert(_nc > 0, "The number of columns in a filter must be > 0");
          static_assert(_stride_y > 0, "The filter stride must be > 0");
          static_assert(_stride_x > 0, "The filter stride must be > 0");
          static_assert(0 <= _padding_y && _padding_y < _nr, "The padding must be smaller than the filter size.");
          static_assert(0 <= _padding_x && _padding_x < _nc, "The padding must be smaller than the filter size.");
  
          con_(
          ) : 
              learning_rate_multiplier(1),
              weight_decay_multiplier(1),
              bias_learning_rate_multiplier(1),
              bias_weight_decay_multiplier(0),
              padding_y_(_padding_y),
              padding_x_(_padding_x)
          {}
  
          long num_filters() const { return _num_filters; }
          long nr() const { return _nr; }
          long nc() const { return _nc; }
          long stride_y() const { return _stride_y; }
          long stride_x() const { return _stride_x; }
          long padding_y() const { return padding_y_; }
          long padding_x() const { return padding_x_; }
  
          double get_learning_rate_multiplier () const  { return learning_rate_multiplier; }
          double get_weight_decay_multiplier () const   { return weight_decay_multiplier; }
          void set_learning_rate_multiplier(double val) { learning_rate_multiplier = val; }
          void set_weight_decay_multiplier(double val)  { weight_decay_multiplier  = val; }
  
          double get_bias_learning_rate_multiplier () const  { return bias_learning_rate_multiplier; }
          double get_bias_weight_decay_multiplier () const   { return bias_weight_decay_multiplier; }
          void set_bias_learning_rate_multiplier(double val) { bias_learning_rate_multiplier = val; }
          void set_bias_weight_decay_multiplier(double val)  { bias_weight_decay_multiplier  = val; }
  
          inline point map_input_to_output (
              point p
          ) const
          {
              p.x() = (p.x()+padding_x()-nc()/2)/stride_x();
              p.y() = (p.y()+padding_y()-nr()/2)/stride_y();
              return p;
          }
  
          inline point map_output_to_input (
              point p
          ) const
          {
              p.x() = p.x()*stride_x() - padding_x() + nc()/2;
              p.y() = p.y()*stride_y() - padding_y() + nr()/2;
              return p;
          }
  
          con_ (
              const con_& item
          ) : 
              params(item.params),
              filters(item.filters),
              biases(item.biases),
              learning_rate_multiplier(item.learning_rate_multiplier),
              weight_decay_multiplier(item.weight_decay_multiplier),
              bias_learning_rate_multiplier(item.bias_learning_rate_multiplier),
              bias_weight_decay_multiplier(item.bias_weight_decay_multiplier),
              padding_y_(item.padding_y_),
              padding_x_(item.padding_x_)
          {
              // this->conv is non-copyable and basically stateless, so we have to write our
              // own copy to avoid trying to copy it and getting an error.
          }
  
          con_& operator= (
              const con_& item
          )
          {
              if (this == &item)
                  return *this;
  
              // this->conv is non-copyable and basically stateless, so we have to write our
              // own copy to avoid trying to copy it and getting an error.
              params = item.params;
              filters = item.filters;
              biases = item.biases;
              padding_y_ = item.padding_y_;
              padding_x_ = item.padding_x_;
              learning_rate_multiplier = item.learning_rate_multiplier;
              weight_decay_multiplier = item.weight_decay_multiplier;
              bias_learning_rate_multiplier = item.bias_learning_rate_multiplier;
              bias_weight_decay_multiplier = item.bias_weight_decay_multiplier;
              return *this;
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& sub)
          {
              long num_inputs = _nr*_nc*sub.get_output().k();
              long num_outputs = _num_filters;
              // allocate params for the filters and also for the filter bias values.
              params.set_size(num_inputs*_num_filters + _num_filters);
  
              dlib::rand rnd(std::rand());
              randomize_parameters(params, num_inputs+num_outputs, rnd);
  
              filters = alias_tensor(_num_filters, sub.get_output().k(), _nr, _nc);
              biases = alias_tensor(1,_num_filters);
  
              // set the initial bias values to zero
              biases(params,filters.size()) = 0;
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              conv(output,
                  sub.get_output(),
                  filters(params,0),
                  _stride_y,
                  _stride_x,
                  padding_y_,
                  padding_x_
                  );
  
              tt::add(1,output,1,biases(params,filters.size()));
          } 
  
          template <typename SUBNET>
          void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad)
          {
              conv.get_gradient_for_data (gradient_input, filters(params,0), sub.get_gradient_input());
              // no point computing the parameter gradients if they won't be used.
              if (learning_rate_multiplier != 0)
              {
                  auto filt = filters(params_grad,0);
                  conv.get_gradient_for_filters (gradient_input, sub.get_output(), filt);
                  auto b = biases(params_grad, filters.size());
                  tt::assign_conv_bias_gradient(b, gradient_input);
              }
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const con_& item, std::ostream& out)
          {
              serialize("con_4", out);
              serialize(item.params, out);
              serialize(_num_filters, out);
              serialize(_nr, out);
              serialize(_nc, out);
              serialize(_stride_y, out);
              serialize(_stride_x, out);
              serialize(item.padding_y_, out);
              serialize(item.padding_x_, out);
              serialize(item.filters, out);
              serialize(item.biases, out);
              serialize(item.learning_rate_multiplier, out);
              serialize(item.weight_decay_multiplier, out);
              serialize(item.bias_learning_rate_multiplier, out);
              serialize(item.bias_weight_decay_multiplier, out);
          }
  
          friend void deserialize(con_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              long num_filters;
              long nr;
              long nc;
              int stride_y;
              int stride_x;
              if (version == "con_4")
              {
                  deserialize(item.params, in);
                  deserialize(num_filters, in);
                  deserialize(nr, in);
                  deserialize(nc, in);
                  deserialize(stride_y, in);
                  deserialize(stride_x, in);
                  deserialize(item.padding_y_, in);
                  deserialize(item.padding_x_, in);
                  deserialize(item.filters, in);
                  deserialize(item.biases, in);
                  deserialize(item.learning_rate_multiplier, in);
                  deserialize(item.weight_decay_multiplier, in);
                  deserialize(item.bias_learning_rate_multiplier, in);
                  deserialize(item.bias_weight_decay_multiplier, in);
                  if (item.padding_y_ != _padding_y) throw serialization_error("Wrong padding_y found while deserializing dlib::con_");
                  if (item.padding_x_ != _padding_x) throw serialization_error("Wrong padding_x found while deserializing dlib::con_");
                  if (num_filters != _num_filters) 
                  {
                      std::ostringstream sout;
                      sout << "Wrong num_filters found while deserializing dlib::con_" << std::endl;
                      sout << "expected " << _num_filters << " but found " << num_filters << std::endl;
                      throw serialization_error(sout.str());
                  }
  
                  if (nr != _nr) throw serialization_error("Wrong nr found while deserializing dlib::con_");
                  if (nc != _nc) throw serialization_error("Wrong nc found while deserializing dlib::con_");
                  if (stride_y != _stride_y) throw serialization_error("Wrong stride_y found while deserializing dlib::con_");
                  if (stride_x != _stride_x) throw serialization_error("Wrong stride_x found while deserializing dlib::con_");
              }
              else
              {
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::con_.");
              }
          }
  
  
          friend std::ostream& operator<<(std::ostream& out, const con_& item)
          {
              out << "con\t ("
                  << "num_filters="<<_num_filters
                  << ", nr="<<_nr
                  << ", nc="<<_nc
                  << ", stride_y="<<_stride_y
                  << ", stride_x="<<_stride_x
                  << ", padding_y="<<item.padding_y_
                  << ", padding_x="<<item.padding_x_
                  << ")";
              out << " learning_rate_mult="<<item.learning_rate_multiplier;
              out << " weight_decay_mult="<<item.weight_decay_multiplier;
              out << " bias_learning_rate_mult="<<item.bias_learning_rate_multiplier;
              out << " bias_weight_decay_mult="<<item.bias_weight_decay_multiplier;
              return out;
          }
  
          friend void to_xml(const con_& item, std::ostream& out)
          {
              out << "<con"
                  << " num_filters='"<<_num_filters<<"'"
                  << " nr='"<<_nr<<"'"
                  << " nc='"<<_nc<<"'"
                  << " stride_y='"<<_stride_y<<"'"
                  << " stride_x='"<<_stride_x<<"'"
                  << " padding_y='"<<item.padding_y_<<"'"
                  << " padding_x='"<<item.padding_x_<<"'"
                  << " learning_rate_mult='"<<item.learning_rate_multiplier<<"'"
                  << " weight_decay_mult='"<<item.weight_decay_multiplier<<"'"
                  << " bias_learning_rate_mult='"<<item.bias_learning_rate_multiplier<<"'"
                  << " bias_weight_decay_mult='"<<item.bias_weight_decay_multiplier<<"'>\n";
              out << mat(item.params);
              out << "</con>";
          }
  
      private:
  
          resizable_tensor params;
          alias_tensor filters, biases;
  
          tt::tensor_conv conv;
          double learning_rate_multiplier;
          double weight_decay_multiplier;
          double bias_learning_rate_multiplier;
          double bias_weight_decay_multiplier;
  
          // These are here only because older versions of con (which you might encounter
          // serialized to disk) used different padding settings.
          int padding_y_;
          int padding_x_;
  
      };
  
      template <
          long num_filters,
          long nr,
          long nc,
          int stride_y,
          int stride_x,
          typename SUBNET
          >
      using con = add_layer<con_<num_filters,nr,nc,stride_y,stride_x>, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      template <
          long _nr,
          long _nc,
          int _stride_y,
          int _stride_x,
          int _padding_y = _stride_y!=1? 0 : _nr/2,
          int _padding_x = _stride_x!=1? 0 : _nc/2
          >
      class max_pool_
      {
          static_assert(_nr >= 0, "The number of rows in a filter must be >= 0");
          static_assert(_nc >= 0, "The number of columns in a filter must be >= 0");
          static_assert(_stride_y > 0, "The filter stride must be > 0");
          static_assert(_stride_x > 0, "The filter stride must be > 0");
          static_assert(0 <= _padding_y && ((_nr==0 && _padding_y == 0) || (_nr!=0 && _padding_y < _nr)), 
              "The padding must be smaller than the filter size, unless the filters size is 0.");
          static_assert(0 <= _padding_x && ((_nc==0 && _padding_x == 0) || (_nc!=0 && _padding_x < _nc)), 
              "The padding must be smaller than the filter size, unless the filters size is 0.");
      public:
  
  
          max_pool_(
          ) :
              padding_y_(_padding_y),
              padding_x_(_padding_x)
          {}
  
          long nr() const { return _nr; }
          long nc() const { return _nc; }
          long stride_y() const { return _stride_y; }
          long stride_x() const { return _stride_x; }
          long padding_y() const { return padding_y_; }
          long padding_x() const { return padding_x_; }
  
          inline point map_input_to_output (
              point p
          ) const
          {
              p.x() = (p.x()+padding_x()-nc()/2)/stride_x();
              p.y() = (p.y()+padding_y()-nr()/2)/stride_y();
              return p;
          }
  
          inline point map_output_to_input (
              point p
          ) const
          {
              p.x() = p.x()*stride_x() - padding_x() + nc()/2;
              p.y() = p.y()*stride_y() - padding_y() + nr()/2;
              return p;
          }
  
          max_pool_ (
              const max_pool_& item
          )  :
              padding_y_(item.padding_y_),
              padding_x_(item.padding_x_)
          {
              // this->mp is non-copyable so we have to write our own copy to avoid trying to
              // copy it and getting an error.
          }
  
          max_pool_& operator= (
              const max_pool_& item
          )
          {
              if (this == &item)
                  return *this;
  
              padding_y_ = item.padding_y_;
              padding_x_ = item.padding_x_;
  
              // this->mp is non-copyable so we have to write our own copy to avoid trying to
              // copy it and getting an error.
              return *this;
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              mp.setup_max_pooling(_nr!=0?_nr:sub.get_output().nr(), 
                                   _nc!=0?_nc:sub.get_output().nc(),
                                   _stride_y, _stride_x, padding_y_, padding_x_);
  
              mp(output, sub.get_output());
          } 
  
          template <typename SUBNET>
          void backward(const tensor& computed_output, const tensor& gradient_input, SUBNET& sub, tensor& /*params_grad*/)
          {
              mp.setup_max_pooling(_nr!=0?_nr:sub.get_output().nr(), 
                                   _nc!=0?_nc:sub.get_output().nc(),
                                   _stride_y, _stride_x, padding_y_, padding_x_);
  
              mp.get_gradient(gradient_input, computed_output, sub.get_output(), sub.get_gradient_input());
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const max_pool_& item, std::ostream& out)
          {
              serialize("max_pool_2", out);
              serialize(_nr, out);
              serialize(_nc, out);
              serialize(_stride_y, out);
              serialize(_stride_x, out);
              serialize(item.padding_y_, out);
              serialize(item.padding_x_, out);
          }
  
          friend void deserialize(max_pool_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              long nr;
              long nc;
              int stride_y;
              int stride_x;
              if (version == "max_pool_2")
              {
                  deserialize(nr, in);
                  deserialize(nc, in);
                  deserialize(stride_y, in);
                  deserialize(stride_x, in);
                  deserialize(item.padding_y_, in);
                  deserialize(item.padding_x_, in);
              }
              else
              {
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::max_pool_.");
              }
  
              if (item.padding_y_ != _padding_y) throw serialization_error("Wrong padding_y found while deserializing dlib::max_pool_");
              if (item.padding_x_ != _padding_x) throw serialization_error("Wrong padding_x found while deserializing dlib::max_pool_");
              if (_nr != nr) throw serialization_error("Wrong nr found while deserializing dlib::max_pool_");
              if (_nc != nc) throw serialization_error("Wrong nc found while deserializing dlib::max_pool_");
              if (_stride_y != stride_y) throw serialization_error("Wrong stride_y found while deserializing dlib::max_pool_");
              if (_stride_x != stride_x) throw serialization_error("Wrong stride_x found while deserializing dlib::max_pool_");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const max_pool_& item)
          {
              out << "max_pool ("
                  << "nr="<<_nr
                  << ", nc="<<_nc
                  << ", stride_y="<<_stride_y
                  << ", stride_x="<<_stride_x
                  << ", padding_y="<<item.padding_y_
                  << ", padding_x="<<item.padding_x_
                  << ")";
              return out;
          }
  
          friend void to_xml(const max_pool_& item, std::ostream& out)
          {
              out << "<max_pool"
                  << " nr='"<<_nr<<"'"
                  << " nc='"<<_nc<<"'"
                  << " stride_y='"<<_stride_y<<"'"
                  << " stride_x='"<<_stride_x<<"'"
                  << " padding_y='"<<item.padding_y_<<"'"
                  << " padding_x='"<<item.padding_x_<<"'"
                  << "/>\n";
          }
  
  
      private:
  
  
          tt::pooling mp;
          resizable_tensor params;
  
          int padding_y_;
          int padding_x_;
      };
  
      template <
          long nr,
          long nc,
          int stride_y,
          int stride_x,
          typename SUBNET
          >
      using max_pool = add_layer<max_pool_<nr,nc,stride_y,stride_x>, SUBNET>;
  
      template <
          typename SUBNET
          >
      using max_pool_everything = add_layer<max_pool_<0,0,1,1>, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      template <
          long _nr,
          long _nc,
          int _stride_y,
          int _stride_x,
          int _padding_y = _stride_y!=1? 0 : _nr/2,
          int _padding_x = _stride_x!=1? 0 : _nc/2
          >
      class avg_pool_
      {
      public:
          static_assert(_nr >= 0, "The number of rows in a filter must be >= 0");
          static_assert(_nc >= 0, "The number of columns in a filter must be >= 0");
          static_assert(_stride_y > 0, "The filter stride must be > 0");
          static_assert(_stride_x > 0, "The filter stride must be > 0");
          static_assert(0 <= _padding_y && ((_nr==0 && _padding_y == 0) || (_nr!=0 && _padding_y < _nr)), 
              "The padding must be smaller than the filter size, unless the filters size is 0.");
          static_assert(0 <= _padding_x && ((_nc==0 && _padding_x == 0) || (_nc!=0 && _padding_x < _nc)), 
              "The padding must be smaller than the filter size, unless the filters size is 0.");
  
          avg_pool_(
          ) :
              padding_y_(_padding_y),
              padding_x_(_padding_x)
          {}
  
          long nr() const { return _nr; }
          long nc() const { return _nc; }
          long stride_y() const { return _stride_y; }
          long stride_x() const { return _stride_x; }
          long padding_y() const { return padding_y_; }
          long padding_x() const { return padding_x_; }
  
          inline point map_input_to_output (
              point p
          ) const
          {
              p.x() = (p.x()+padding_x()-nc()/2)/stride_x();
              p.y() = (p.y()+padding_y()-nr()/2)/stride_y();
              return p;
          }
  
          inline point map_output_to_input (
              point p
          ) const
          {
              p.x() = p.x()*stride_x() - padding_x() + nc()/2;
              p.y() = p.y()*stride_y() - padding_y() + nr()/2;
              return p;
          }
  
          avg_pool_ (
              const avg_pool_& item
          )  :
              padding_y_(item.padding_y_),
              padding_x_(item.padding_x_)
          {
              // this->ap is non-copyable so we have to write our own copy to avoid trying to
              // copy it and getting an error.
          }
  
          avg_pool_& operator= (
              const avg_pool_& item
          )
          {
              if (this == &item)
                  return *this;
  
              padding_y_ = item.padding_y_;
              padding_x_ = item.padding_x_;
  
              // this->ap is non-copyable so we have to write our own copy to avoid trying to
              // copy it and getting an error.
              return *this;
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              ap.setup_avg_pooling(_nr!=0?_nr:sub.get_output().nr(), 
                                   _nc!=0?_nc:sub.get_output().nc(),
                                   _stride_y, _stride_x, padding_y_, padding_x_);
  
              ap(output, sub.get_output());
          } 
  
          template <typename SUBNET>
          void backward(const tensor& computed_output, const tensor& gradient_input, SUBNET& sub, tensor& /*params_grad*/)
          {
              ap.setup_avg_pooling(_nr!=0?_nr:sub.get_output().nr(), 
                                   _nc!=0?_nc:sub.get_output().nc(),
                                   _stride_y, _stride_x, padding_y_, padding_x_);
  
              ap.get_gradient(gradient_input, computed_output, sub.get_output(), sub.get_gradient_input());
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const avg_pool_& item, std::ostream& out)
          {
              serialize("avg_pool_2", out);
              serialize(_nr, out);
              serialize(_nc, out);
              serialize(_stride_y, out);
              serialize(_stride_x, out);
              serialize(item.padding_y_, out);
              serialize(item.padding_x_, out);
          }
  
          friend void deserialize(avg_pool_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
  
              long nr;
              long nc;
              int stride_y;
              int stride_x;
              if (version == "avg_pool_2")
              {
                  deserialize(nr, in);
                  deserialize(nc, in);
                  deserialize(stride_y, in);
                  deserialize(stride_x, in);
                  deserialize(item.padding_y_, in);
                  deserialize(item.padding_x_, in);
              }
              else
              {
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::avg_pool_.");
              }
  
              if (item.padding_y_ != _padding_y) throw serialization_error("Wrong padding_y found while deserializing dlib::avg_pool_");
              if (item.padding_x_ != _padding_x) throw serialization_error("Wrong padding_x found while deserializing dlib::avg_pool_");
              if (_nr != nr) throw serialization_error("Wrong nr found while deserializing dlib::avg_pool_");
              if (_nc != nc) throw serialization_error("Wrong nc found while deserializing dlib::avg_pool_");
              if (_stride_y != stride_y) throw serialization_error("Wrong stride_y found while deserializing dlib::avg_pool_");
              if (_stride_x != stride_x) throw serialization_error("Wrong stride_x found while deserializing dlib::avg_pool_");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const avg_pool_& item)
          {
              out << "avg_pool ("
                  << "nr="<<_nr
                  << ", nc="<<_nc
                  << ", stride_y="<<_stride_y
                  << ", stride_x="<<_stride_x
                  << ", padding_y="<<item.padding_y_
                  << ", padding_x="<<item.padding_x_
                  << ")";
              return out;
          }
  
          friend void to_xml(const avg_pool_& item, std::ostream& out)
          {
              out << "<avg_pool"
                  << " nr='"<<_nr<<"'"
                  << " nc='"<<_nc<<"'"
                  << " stride_y='"<<_stride_y<<"'"
                  << " stride_x='"<<_stride_x<<"'"
                  << " padding_y='"<<item.padding_y_<<"'"
                  << " padding_x='"<<item.padding_x_<<"'"
                  << "/>\n";
          }
      private:
  
          tt::pooling ap;
          resizable_tensor params;
  
          int padding_y_;
          int padding_x_;
      };
  
      template <
          long nr,
          long nc,
          int stride_y,
          int stride_x,
          typename SUBNET
          >
      using avg_pool = add_layer<avg_pool_<nr,nc,stride_y,stride_x>, SUBNET>;
  
      template <
          typename SUBNET
          >
      using avg_pool_everything = add_layer<avg_pool_<0,0,1,1>, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      enum layer_mode
      {
          CONV_MODE = 0,
          FC_MODE = 1
      };
  
      const double DEFAULT_BATCH_NORM_EPS = 0.0001;
  
      template <
          layer_mode mode
          >
      class bn_
      {
      public:
          explicit bn_(
              unsigned long window_size,
              double eps_ = DEFAULT_BATCH_NORM_EPS
          ) : 
              num_updates(0), 
              running_stats_window_size(window_size),
              learning_rate_multiplier(1),
              weight_decay_multiplier(0),
              bias_learning_rate_multiplier(1),
              bias_weight_decay_multiplier(1),
              eps(eps_)
          {
              DLIB_CASSERT(window_size > 0, "The batch normalization running stats window size can't be 0.");
          }
  
          bn_() : bn_(100) {}
  
          layer_mode get_mode() const { return mode; }
          unsigned long get_running_stats_window_size () const { return running_stats_window_size; }
          void set_running_stats_window_size (unsigned long new_window_size ) 
          { 
              DLIB_CASSERT(new_window_size > 0, "The batch normalization running stats window size can't be 0.");
              running_stats_window_size = new_window_size; 
          }
          double get_eps() const { return eps; }
  
          double get_learning_rate_multiplier () const  { return learning_rate_multiplier; }
          double get_weight_decay_multiplier () const   { return weight_decay_multiplier; }
          void set_learning_rate_multiplier(double val) { learning_rate_multiplier = val; }
          void set_weight_decay_multiplier(double val)  { weight_decay_multiplier  = val; }
  
          double get_bias_learning_rate_multiplier () const  { return bias_learning_rate_multiplier; }
          double get_bias_weight_decay_multiplier () const   { return bias_weight_decay_multiplier; }
          void set_bias_learning_rate_multiplier(double val) { bias_learning_rate_multiplier = val; }
          void set_bias_weight_decay_multiplier(double val)  { bias_weight_decay_multiplier  = val; }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
  
          template <typename SUBNET>
          void setup (const SUBNET& sub)
          {
              if (mode == FC_MODE)
              {
                  gamma = alias_tensor(1,
                                  sub.get_output().k(),
                                  sub.get_output().nr(),
                                  sub.get_output().nc());
              }
              else
              {
                  gamma = alias_tensor(1, sub.get_output().k());
              }
              beta = gamma;
  
              params.set_size(gamma.size()+beta.size());
  
              gamma(params,0) = 1;
              beta(params,gamma.size()) = 0;
  
              running_means.copy_size(gamma(params,0));
              running_variances.copy_size(gamma(params,0));
              running_means = 0;
              running_variances = 1;
              num_updates = 0;
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              auto g = gamma(params,0);
              auto b = beta(params,gamma.size());
              if (sub.get_output().num_samples() > 1)
              {
                  const double decay = 1.0 - num_updates/(num_updates+1.0);
                  ++num_updates;
                  if (num_updates > running_stats_window_size)
                      num_updates = running_stats_window_size;
  
                  if (mode == FC_MODE)
                      tt::batch_normalize(eps, output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b);
                  else 
                      tt::batch_normalize_conv(eps, output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b);
              }
              else // we are running in testing mode so we just linearly scale the input tensor.
              {
                  if (mode == FC_MODE)
                      tt::batch_normalize_inference(eps, output, sub.get_output(), g, b, running_means, running_variances);
                  else
                      tt::batch_normalize_conv_inference(eps, output, sub.get_output(), g, b, running_means, running_variances);
              }
          } 
  
          template <typename SUBNET>
          void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad)
          {
              auto g = gamma(params,0);
              auto g_grad = gamma(params_grad, 0);
              auto b_grad = beta(params_grad, gamma.size());
              if (mode == FC_MODE)
                  tt::batch_normalize_gradient(eps, gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
              else
                  tt::batch_normalize_conv_gradient(eps, gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const bn_& item, std::ostream& out)
          {
              if (mode == CONV_MODE)
                  serialize("bn_con2", out);
              else // if FC_MODE
                  serialize("bn_fc2", out);
              serialize(item.params, out);
              serialize(item.gamma, out);
              serialize(item.beta, out);
              serialize(item.means, out);
              serialize(item.invstds, out);
              serialize(item.running_means, out);
              serialize(item.running_variances, out);
              serialize(item.num_updates, out);
              serialize(item.running_stats_window_size, out);
              serialize(item.learning_rate_multiplier, out);
              serialize(item.weight_decay_multiplier, out);
              serialize(item.bias_learning_rate_multiplier, out);
              serialize(item.bias_weight_decay_multiplier, out);
              serialize(item.eps, out);
          }
  
          friend void deserialize(bn_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (mode == CONV_MODE) 
              {
                  if (version != "bn_con2")
                      throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::bn_.");
              }
              else // must be in FC_MODE
              {
                  if (version != "bn_fc2")
                      throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::bn_.");
              }
  
              deserialize(item.params, in);
              deserialize(item.gamma, in);
              deserialize(item.beta, in);
              deserialize(item.means, in);
              deserialize(item.invstds, in);
              deserialize(item.running_means, in);
              deserialize(item.running_variances, in);
              deserialize(item.num_updates, in);
              deserialize(item.running_stats_window_size, in);
              deserialize(item.learning_rate_multiplier, in);
              deserialize(item.weight_decay_multiplier, in);
              deserialize(item.bias_learning_rate_multiplier, in);
              deserialize(item.bias_weight_decay_multiplier, in);
              deserialize(item.eps, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const bn_& item)
          {
              if (mode == CONV_MODE)
                  out << "bn_con  ";
              else
                  out << "bn_fc   ";
              out << " eps="<<item.eps;
              out << " running_stats_window_size="<<item.running_stats_window_size;
              out << " learning_rate_mult="<<item.learning_rate_multiplier;
              out << " weight_decay_mult="<<item.weight_decay_multiplier;
              out << " bias_learning_rate_mult="<<item.bias_learning_rate_multiplier;
              out << " bias_weight_decay_mult="<<item.bias_weight_decay_multiplier;
              return out;
          }
  
          friend void to_xml(const bn_& item, std::ostream& out)
          {
              if (mode==CONV_MODE)
                  out << "<bn_con";
              else
                  out << "<bn_fc";
  
              out << " eps='"<<item.eps<<"'";
              out << " running_stats_window_size='"<<item.running_stats_window_size<<"'";
              out << " learning_rate_mult='"<<item.learning_rate_multiplier<<"'";
              out << " weight_decay_mult='"<<item.weight_decay_multiplier<<"'";
              out << " bias_learning_rate_mult='"<<item.bias_learning_rate_multiplier<<"'";
              out << " bias_weight_decay_mult='"<<item.bias_weight_decay_multiplier<<"'";
              out << ">\n";
  
              out << mat(item.params);
  
              if (mode==CONV_MODE)
                  out << "</bn_con>\n";
              else
                  out << "</bn_fc>\n";
          }
  
      private:
  
          friend class affine_;
  
          resizable_tensor params;
          alias_tensor gamma, beta;
          resizable_tensor means, running_means;
          resizable_tensor invstds, running_variances;
          unsigned long num_updates;
          unsigned long running_stats_window_size;
          double learning_rate_multiplier;
          double weight_decay_multiplier;
          double bias_learning_rate_multiplier;
          double bias_weight_decay_multiplier;
          double eps;
      };
  
      template <typename SUBNET>
      using bn_con = add_layer<bn_<CONV_MODE>, SUBNET>;
      template <typename SUBNET>
      using bn_fc = add_layer<bn_<FC_MODE>, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      namespace impl
      {
          class visitor_bn_running_stats_window_size
          {
          public:
  
              visitor_bn_running_stats_window_size(unsigned long new_window_size_) : new_window_size(new_window_size_) {}
  
              template <typename T>
              void set_window_size(T&) const
              {
                  // ignore other layer detail types
              }
  
              template < layer_mode mode >
              void set_window_size(bn_<mode>& l) const
              {
                  l.set_running_stats_window_size(new_window_size);
              }
  
              template<typename input_layer_type>
              void operator()(size_t , input_layer_type& )  const
              {
                  // ignore other layers
              }
  
              template <typename T, typename U, typename E>
              void operator()(size_t , add_layer<T,U,E>& l)  const
              {
                  set_window_size(l.layer_details());
              }
  
          private:
  
              unsigned long new_window_size;
          };
      }
  
      template <typename net_type>
      void set_all_bn_running_stats_window_sizes (
          net_type& net,
          unsigned long new_window_size
      )
      {
          visit_layers(net, impl::visitor_bn_running_stats_window_size(new_window_size));
      }
  
  // ----------------------------------------------------------------------------------------
  // ----------------------------------------------------------------------------------------
  
      enum fc_bias_mode
      {
          FC_HAS_BIAS = 0,
          FC_NO_BIAS = 1
      };
  
      struct num_fc_outputs
      {
          num_fc_outputs(unsigned long n) : num_outputs(n) {}
          unsigned long num_outputs;
      };
  
      template <
          unsigned long num_outputs_,
          fc_bias_mode bias_mode
          >
      class fc_
      {
          static_assert(num_outputs_ > 0, "The number of outputs from a fc_ layer must be > 0");
  
      public:
          fc_(num_fc_outputs o) : num_outputs(o.num_outputs), num_inputs(0),
              learning_rate_multiplier(1),
              weight_decay_multiplier(1),
              bias_learning_rate_multiplier(1),
              bias_weight_decay_multiplier(0)
          {}
  
          fc_() : fc_(num_fc_outputs(num_outputs_)) {}
  
          double get_learning_rate_multiplier () const  { return learning_rate_multiplier; }
          double get_weight_decay_multiplier () const   { return weight_decay_multiplier; }
          void set_learning_rate_multiplier(double val) { learning_rate_multiplier = val; }
          void set_weight_decay_multiplier(double val)  { weight_decay_multiplier  = val; }
  
          double get_bias_learning_rate_multiplier () const  { return bias_learning_rate_multiplier; }
          double get_bias_weight_decay_multiplier () const   { return bias_weight_decay_multiplier; }
          void set_bias_learning_rate_multiplier(double val) { bias_learning_rate_multiplier = val; }
          void set_bias_weight_decay_multiplier(double val)  { bias_weight_decay_multiplier  = val; }
  
          unsigned long get_num_outputs (
          ) const { return num_outputs; }
  
          fc_bias_mode get_bias_mode (
          ) const { return bias_mode; }
  
          template <typename SUBNET>
          void setup (const SUBNET& sub)
          {
              num_inputs = sub.get_output().nr()*sub.get_output().nc()*sub.get_output().k();
              if (bias_mode == FC_HAS_BIAS)
                  params.set_size(num_inputs+1, num_outputs);
              else
                  params.set_size(num_inputs, num_outputs);
  
              dlib::rand rnd(std::rand());
              randomize_parameters(params, num_inputs+num_outputs, rnd);
  
              weights = alias_tensor(num_inputs, num_outputs);
  
              if (bias_mode == FC_HAS_BIAS)
              {
                  biases = alias_tensor(1,num_outputs);
                  // set the initial bias values to zero
                  biases(params,weights.size()) = 0;
              }
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              output.set_size(sub.get_output().num_samples(), num_outputs);
  
              auto w = weights(params, 0);
              tt::gemm(0,output, 1,sub.get_output(),false, w,false);
              if (bias_mode == FC_HAS_BIAS)
              {
                  auto b = biases(params, weights.size());
                  tt::add(1,output,1,b);
              }
          } 
  
          template <typename SUBNET>
          void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad)
          {
              // no point computing the parameter gradients if they won't be used.
              if (learning_rate_multiplier != 0)
              {
                  // compute the gradient of the weight parameters.  
                  auto pw = weights(params_grad, 0);
                  tt::gemm(0,pw, 1,sub.get_output(),true, gradient_input,false);
  
                  if (bias_mode == FC_HAS_BIAS)
                  {
                      // compute the gradient of the bias parameters.  
                      auto pb = biases(params_grad, weights.size());
                      tt::assign_bias_gradient(pb, gradient_input);
                  }
              }
  
              // compute the gradient for the data
              auto w = weights(params, 0);
              tt::gemm(1,sub.get_gradient_input(), 1,gradient_input,false, w,true);
          }
  
          alias_tensor_instance get_weights()
          {
              return weights(params, 0);
          }
  
          alias_tensor_const_instance get_weights() const
          {
              return weights(params, 0);
          }
  
          alias_tensor_instance get_biases()
          {
              static_assert(bias_mode == FC_HAS_BIAS, "This fc_ layer doesn't have a bias vector "
                  "to be retrieved, as per template parameter 'bias_mode'.");
              return biases(params, weights.size());
          }
  
          alias_tensor_const_instance get_biases() const
          {
              static_assert(bias_mode == FC_HAS_BIAS, "This fc_ layer doesn't have a bias vector "
                  "to be retrieved, as per template parameter 'bias_mode'.");
              return biases(params, weights.size());
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const fc_& item, std::ostream& out)
          {
              serialize("fc_2", out);
              serialize(item.num_outputs, out);
              serialize(item.num_inputs, out);
              serialize(item.params, out);
              serialize(item.weights, out);
              serialize(item.biases, out);
              serialize((int)bias_mode, out);
              serialize(item.learning_rate_multiplier, out);
              serialize(item.weight_decay_multiplier, out);
              serialize(item.bias_learning_rate_multiplier, out);
              serialize(item.bias_weight_decay_multiplier, out);
          }
  
          friend void deserialize(fc_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "fc_2")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::fc_.");
  
              deserialize(item.num_outputs, in);
              deserialize(item.num_inputs, in);
              deserialize(item.params, in);
              deserialize(item.weights, in);
              deserialize(item.biases, in);
              int bmode = 0;
              deserialize(bmode, in);
              if (bias_mode != (fc_bias_mode)bmode) throw serialization_error("Wrong fc_bias_mode found while deserializing dlib::fc_");
              deserialize(item.learning_rate_multiplier, in);
              deserialize(item.weight_decay_multiplier, in);
              deserialize(item.bias_learning_rate_multiplier, in);
              deserialize(item.bias_weight_decay_multiplier, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const fc_& item)
          {
              if (bias_mode == FC_HAS_BIAS)
              {
                  out << "fc\t ("
                      << "num_outputs="<<item.num_outputs
                      << ")";
                  out << " learning_rate_mult="<<item.learning_rate_multiplier;
                  out << " weight_decay_mult="<<item.weight_decay_multiplier;
                  out << " bias_learning_rate_mult="<<item.bias_learning_rate_multiplier;
                  out << " bias_weight_decay_mult="<<item.bias_weight_decay_multiplier;
              }
              else
              {
                  out << "fc_no_bias ("
                      << "num_outputs="<<item.num_outputs
                      << ")";
                  out << " learning_rate_mult="<<item.learning_rate_multiplier;
                  out << " weight_decay_mult="<<item.weight_decay_multiplier;
              }
              return out;
          }
  
          friend void to_xml(const fc_& item, std::ostream& out)
          {
              if (bias_mode==FC_HAS_BIAS)
              {
                  out << "<fc"
                      << " num_outputs='"<<item.num_outputs<<"'"
                      << " learning_rate_mult='"<<item.learning_rate_multiplier<<"'"
                      << " weight_decay_mult='"<<item.weight_decay_multiplier<<"'"
                      << " bias_learning_rate_mult='"<<item.bias_learning_rate_multiplier<<"'"
                      << " bias_weight_decay_mult='"<<item.bias_weight_decay_multiplier<<"'";
                  out << ">\n";
                  out << mat(item.params);
                  out << "</fc>\n";
              }
              else
              {
                  out << "<fc_no_bias"
                      << " num_outputs='"<<item.num_outputs<<"'"
                      << " learning_rate_mult='"<<item.learning_rate_multiplier<<"'"
                      << " weight_decay_mult='"<<item.weight_decay_multiplier<<"'";
                  out << ">\n";
                  out << mat(item.params);
                  out << "</fc_no_bias>\n";
              }
          }
  
      private:
  
          unsigned long num_outputs;
          unsigned long num_inputs;
          resizable_tensor params;
          alias_tensor weights, biases;
          double learning_rate_multiplier;
          double weight_decay_multiplier;
          double bias_learning_rate_multiplier;
          double bias_weight_decay_multiplier;
      };
  
      template <
          unsigned long num_outputs,
          typename SUBNET
          >
      using fc = add_layer<fc_<num_outputs,FC_HAS_BIAS>, SUBNET>;
  
      template <
          unsigned long num_outputs,
          typename SUBNET
          >
      using fc_no_bias = add_layer<fc_<num_outputs,FC_NO_BIAS>, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class dropout_
      {
      public:
          explicit dropout_(
              float drop_rate_ = 0.5
          ) :
              drop_rate(drop_rate_),
              rnd(std::rand())
          {
              DLIB_CASSERT(0 <= drop_rate && drop_rate <= 1);
          }
  
          // We have to add a copy constructor and assignment operator because the rnd object
          // is non-copyable.
          dropout_(
              const dropout_& item
          ) : drop_rate(item.drop_rate), mask(item.mask), rnd(std::rand())
          {}
  
          dropout_& operator= (
              const dropout_& item
          )
          {
              if (this == &item)
                  return *this;
  
              drop_rate = item.drop_rate;
              mask = item.mask;
              return *this;
          }
  
          float get_drop_rate (
          ) const { return drop_rate; }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              // create a random mask and use it to filter the data
              mask.copy_size(input);
              rnd.fill_uniform(mask);
              tt::threshold(mask, drop_rate);
              tt::multiply(false, output, input, mask);
          } 
  
          void backward_inplace(
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& /*params_grad*/
          )
          {
              if (is_same_object(gradient_input, data_grad))
                  tt::multiply(false, data_grad, mask, gradient_input);
              else
                  tt::multiply(true, data_grad, mask, gradient_input);
          }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const dropout_& item, std::ostream& out)
          {
              serialize("dropout_", out);
              serialize(item.drop_rate, out);
              serialize(item.mask, out);
          }
  
          friend void deserialize(dropout_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "dropout_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::dropout_.");
              deserialize(item.drop_rate, in);
              deserialize(item.mask, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const dropout_& item)
          {
              out << "dropout\t ("
                  << "drop_rate="<<item.drop_rate
                  << ")";
              return out;
          }
  
          friend void to_xml(const dropout_& item, std::ostream& out)
          {
              out << "<dropout"
                  << " drop_rate='"<<item.drop_rate<<"'";
              out << "/>\n";
          }
  
      private:
          float drop_rate;
          resizable_tensor mask;
  
          tt::tensor_rand rnd;
          resizable_tensor params; // unused
      };
  
  
      template <typename SUBNET>
      using dropout = add_layer<dropout_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class multiply_
      {
      public:
          explicit multiply_(
              float val_ = 0.5
          ) :
              val(val_)
          {
          }
  
          multiply_ (
              const dropout_& item
          ) : val(1-item.get_drop_rate()) {}
  
          float get_multiply_value (
          ) const { return val; }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::affine_transform(output, input, val);
          } 
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          void backward_inplace(
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& /*params_grad*/
          )
          {
              if (is_same_object(gradient_input, data_grad))
                  tt::affine_transform(data_grad, gradient_input, val);
              else
                  tt::affine_transform(data_grad, data_grad, gradient_input, 1, val);
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const multiply_& item, std::ostream& out)
          {
              serialize("multiply_", out);
              serialize(item.val, out);
          }
  
          friend void deserialize(multiply_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version == "dropout_")
              {
                  // Since we can build a multiply_ from a dropout_ we check if that's what
                  // is in the stream and if so then just convert it right here.
                  unserialize sin(version, in);
                  dropout_ temp;
                  deserialize(temp, sin);
                  item = temp;
                  return;
              }
  
              if (version != "multiply_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::multiply_.");
              deserialize(item.val, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const multiply_& item)
          {
              out << "multiply ("
                  << "val="<<item.val
                  << ")";
              return out;
          }
  
          friend void to_xml(const multiply_& item, std::ostream& out)
          {
              out << "<multiply"
                  << " val='"<<item.val<<"'";
              out << "/>\n";
          }
      private:
          float val;
          resizable_tensor params; // unused
      };
  
      template <typename SUBNET>
      using multiply = add_layer<multiply_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class affine_
      {
      public:
          affine_(
          ) : mode(FC_MODE)
          {
          }
  
          affine_(
              layer_mode mode_
          ) : mode(mode_)
          {
          }
  
          template <
              layer_mode bnmode
              >
          affine_(
              const bn_<bnmode>& item
          )
          {
              gamma = item.gamma;
              beta = item.beta;
              mode = bnmode;
  
              params.copy_size(item.params);
  
              auto g = gamma(params,0);
              auto b = beta(params,gamma.size());
              
              resizable_tensor temp(item.params);
              auto sg = gamma(temp,0);
              auto sb = beta(temp,gamma.size());
  
              g = pointwise_multiply(mat(sg), 1.0f/sqrt(mat(item.running_variances)+item.get_eps()));
              b = mat(sb) - pointwise_multiply(mat(g), mat(item.running_means));
          }
  
          layer_mode get_mode() const { return mode; }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          template <typename SUBNET>
          void setup (const SUBNET& sub)
          {
              if (mode == FC_MODE)
              {
                  gamma = alias_tensor(1,
                                  sub.get_output().k(),
                                  sub.get_output().nr(),
                                  sub.get_output().nc());
              }
              else
              {
                  gamma = alias_tensor(1, sub.get_output().k());
              }
              beta = gamma;
  
              params.set_size(gamma.size()+beta.size());
  
              gamma(params,0) = 1;
              beta(params,gamma.size()) = 0;
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              auto g = gamma(params,0);
              auto b = beta(params,gamma.size());
              if (mode == FC_MODE)
                  tt::affine_transform(output, input, g, b);
              else
                  tt::affine_transform_conv(output, input, g, b);
          } 
  
          void backward_inplace(
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& /*params_grad*/
          )
          {
              auto g = gamma(params,0);
              auto b = beta(params,gamma.size());
  
              // We are computing the gradient of dot(gradient_input, computed_output*g + b)
              if (mode == FC_MODE)
              {
                  if (is_same_object(gradient_input, data_grad))
                      tt::multiply(false, data_grad, gradient_input, g);
                  else
                      tt::multiply(true, data_grad, gradient_input, g);
              }
              else
              {
                  if (is_same_object(gradient_input, data_grad))
                      tt::multiply_conv(false, data_grad, gradient_input, g);
                  else
                      tt::multiply_conv(true, data_grad, gradient_input, g);
              }
          }
  
          const tensor& get_layer_params() const { return empty_params; }
          tensor& get_layer_params() { return empty_params; }
  
          friend void serialize(const affine_& item, std::ostream& out)
          {
              serialize("affine_", out);
              serialize(item.params, out);
              serialize(item.gamma, out);
              serialize(item.beta, out);
              serialize((int)item.mode, out);
          }
  
          friend void deserialize(affine_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version == "bn_con2")
              {
                  // Since we can build an affine_ from a bn_ we check if that's what is in
                  // the stream and if so then just convert it right here.
                  unserialize sin(version, in);
                  bn_<CONV_MODE> temp;
                  deserialize(temp, sin);
                  item = temp;
                  return;
              }
              else if (version == "bn_fc2")
              {
                  // Since we can build an affine_ from a bn_ we check if that's what is in
                  // the stream and if so then just convert it right here.
                  unserialize sin(version, in);
                  bn_<FC_MODE> temp;
                  deserialize(temp, sin);
                  item = temp;
                  return;
              }
  
              if (version != "affine_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::affine_.");
              deserialize(item.params, in);
              deserialize(item.gamma, in);
              deserialize(item.beta, in);
              int mode;
              deserialize(mode, in);
              item.mode = (layer_mode)mode;
          }
  
          friend std::ostream& operator<<(std::ostream& out, const affine_& )
          {
              out << "affine";
              return out;
          }
  
          friend void to_xml(const affine_& item, std::ostream& out)
          {
              out << "<affine";
              if (item.mode==CONV_MODE)
                  out << " mode='conv'";
              else
                  out << " mode='fc'";
              out << ">\n";
              out << mat(item.params);
              out << "</affine>\n";
          }
  
      private:
          resizable_tensor params, empty_params; 
          alias_tensor gamma, beta;
          layer_mode mode;
      };
  
      template <typename SUBNET>
      using affine = add_layer<affine_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      template <
          template<typename> class tag
          >
      class add_prev_
      {
      public:
          const static unsigned long id = tag_id<tag>::id;
  
          add_prev_() 
          {
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              auto&& t1 = sub.get_output();
              auto&& t2 = layer<tag>(sub).get_output();
              output.set_size(std::max(t1.num_samples(),t2.num_samples()),
                              std::max(t1.k(),t2.k()),
                              std::max(t1.nr(),t2.nr()),
                              std::max(t1.nc(),t2.nc()));
              tt::add(output, t1, t2);
          }
  
          template <typename SUBNET>
          void backward(const tensor& gradient_input, SUBNET& sub, tensor& /*params_grad*/)
          {
              // The gradient just flows backwards to the two layers that forward() added
              // together.
              tt::add(sub.get_gradient_input(), sub.get_gradient_input(), gradient_input);
              tt::add(layer<tag>(sub).get_gradient_input(), layer<tag>(sub).get_gradient_input(), gradient_input);
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const add_prev_& , std::ostream& out)
          {
              serialize("add_prev_", out);
          }
  
          friend void deserialize(add_prev_& , std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "add_prev_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::add_prev_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const add_prev_& item)
          {
              out << "add_prev"<<id;
              return out;
          }
  
          friend void to_xml(const add_prev_& item, std::ostream& out)
          {
              out << "<add_prev tag='"<<id<<"'/>\n";
          }
  
      private:
          resizable_tensor params;
      };
  
      template <
          template<typename> class tag,
          typename SUBNET
          >
      using add_prev = add_layer<add_prev_<tag>, SUBNET>;
  
      template <typename SUBNET> using add_prev1  = add_prev<tag1, SUBNET>;
      template <typename SUBNET> using add_prev2  = add_prev<tag2, SUBNET>;
      template <typename SUBNET> using add_prev3  = add_prev<tag3, SUBNET>;
      template <typename SUBNET> using add_prev4  = add_prev<tag4, SUBNET>;
      template <typename SUBNET> using add_prev5  = add_prev<tag5, SUBNET>;
      template <typename SUBNET> using add_prev6  = add_prev<tag6, SUBNET>;
      template <typename SUBNET> using add_prev7  = add_prev<tag7, SUBNET>;
      template <typename SUBNET> using add_prev8  = add_prev<tag8, SUBNET>;
      template <typename SUBNET> using add_prev9  = add_prev<tag9, SUBNET>;
      template <typename SUBNET> using add_prev10 = add_prev<tag10, SUBNET>;
  
      using add_prev1_  = add_prev_<tag1>;
      using add_prev2_  = add_prev_<tag2>;
      using add_prev3_  = add_prev_<tag3>;
      using add_prev4_  = add_prev_<tag4>;
      using add_prev5_  = add_prev_<tag5>;
      using add_prev6_  = add_prev_<tag6>;
      using add_prev7_  = add_prev_<tag7>;
      using add_prev8_  = add_prev_<tag8>;
      using add_prev9_  = add_prev_<tag9>;
      using add_prev10_ = add_prev_<tag10>;
  
  // ----------------------------------------------------------------------------------------
  
      class relu_
      {
      public:
          relu_() 
          {
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::relu(output, input);
          } 
  
          void backward_inplace(
              const tensor& computed_output,
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& 
          )
          {
              tt::relu_gradient(data_grad, computed_output, gradient_input);
          }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const relu_& , std::ostream& out)
          {
              serialize("relu_", out);
          }
  
          friend void deserialize(relu_& , std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "relu_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::relu_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const relu_& )
          {
              out << "relu";
              return out;
          }
  
          friend void to_xml(const relu_& /*item*/, std::ostream& out)
          {
              out << "<relu/>\n";
          }
  
      private:
          resizable_tensor params;
      };
  
  
      template <typename SUBNET>
      using relu = add_layer<relu_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class prelu_
      {
      public:
          explicit prelu_(
              float initial_param_value_ = 0.25
          ) : initial_param_value(initial_param_value_)
          {
          }
  
          float get_initial_param_value (
          ) const { return initial_param_value; }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
              params.set_size(1);
              params = initial_param_value;
          }
  
          template <typename SUBNET>
          void forward(
              const SUBNET& sub, 
              resizable_tensor& data_output
          )
          {
              data_output.copy_size(sub.get_output());
              tt::prelu(data_output, sub.get_output(), params);
          }
  
          template <typename SUBNET>
          void backward(
              const tensor& gradient_input, 
              SUBNET& sub, 
              tensor& params_grad
          )
          {
              tt::prelu_gradient(sub.get_gradient_input(), sub.get_output(), 
                  gradient_input, params, params_grad);
          }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const prelu_& item, std::ostream& out)
          {
              serialize("prelu_", out);
              serialize(item.params, out);
              serialize(item.initial_param_value, out);
          }
  
          friend void deserialize(prelu_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "prelu_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::prelu_.");
              deserialize(item.params, in);
              deserialize(item.initial_param_value, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const prelu_& item)
          {
              out << "prelu\t ("
                  << "initial_param_value="<<item.initial_param_value
                  << ")";
              return out;
          }
  
          friend void to_xml(const prelu_& item, std::ostream& out)
          {
              out << "<prelu initial_param_value='"<<item.initial_param_value<<"'>\n";
              out << mat(item.params);
              out << "</prelu>\n";
          }
  
      private:
          resizable_tensor params;
          float initial_param_value;
      };
  
      template <typename SUBNET>
      using prelu = add_layer<prelu_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class sig_
      {
      public:
          sig_() 
          {
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::sigmoid(output, input);
          } 
  
          void backward_inplace(
              const tensor& computed_output,
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& 
          )
          {
              tt::sigmoid_gradient(data_grad, computed_output, gradient_input);
          }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const sig_& , std::ostream& out)
          {
              serialize("sig_", out);
          }
  
          friend void deserialize(sig_& , std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "sig_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::sig_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const sig_& )
          {
              out << "sig";
              return out;
          }
  
          friend void to_xml(const sig_& /*item*/, std::ostream& out)
          {
              out << "<sig/>\n";
          }
  
  
      private:
          resizable_tensor params;
      };
  
  
      template <typename SUBNET>
      using sig = add_layer<sig_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class htan_
      {
      public:
          htan_() 
          {
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          inline point map_input_to_output (const point& p) const { return p; }
          inline point map_output_to_input (const point& p) const { return p; }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::tanh(output, input);
          } 
  
          void backward_inplace(
              const tensor& computed_output,
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& 
          )
          {
              tt::tanh_gradient(data_grad, computed_output, gradient_input);
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const htan_& , std::ostream& out)
          {
              serialize("htan_", out);
          }
  
          friend void deserialize(htan_& , std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "htan_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::htan_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const htan_& )
          {
              out << "htan";
              return out;
          }
  
          friend void to_xml(const htan_& /*item*/, std::ostream& out)
          {
              out << "<htan/>\n";
          }
  
  
      private:
          resizable_tensor params;
      };
  
  
      template <typename SUBNET>
      using htan = add_layer<htan_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
      class softmax_
      {
      public:
          softmax_() 
          {
          }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::softmax(output, input);
          } 
  
          void backward_inplace(
              const tensor& computed_output,
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& 
          )
          {
              tt::softmax_gradient(data_grad, computed_output, gradient_input);
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const softmax_& , std::ostream& out)
          {
              serialize("softmax_", out);
          }
  
          friend void deserialize(softmax_& , std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "softmax_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::softmax_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const softmax_& )
          {
              out << "softmax";
              return out;
          }
  
          friend void to_xml(const softmax_& /*item*/, std::ostream& out)
          {
              out << "<softmax/>\n";
          }
  
      private:
          resizable_tensor params;
      };
  
      template <typename SUBNET>
      using softmax = add_layer<softmax_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
      namespace impl
      {
          template <template<typename> class TAG_TYPE, template<typename> class... TAG_TYPES>
          struct concat_helper_impl{
  
              constexpr static size_t tag_count() {return 1 + concat_helper_impl<TAG_TYPES...>::tag_count();}
              static void list_tags(std::ostream& out)
              {
                  out << tag_id<TAG_TYPE>::id << (tag_count() > 1 ? "," : "");
                  concat_helper_impl<TAG_TYPES...>::list_tags(out);
              }
  
              template<typename SUBNET>
              static void resize_out(resizable_tensor& out, const SUBNET& sub, long sum_k)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_output();
                  concat_helper_impl<TAG_TYPES...>::resize_out(out, sub, sum_k + t.k());
              }
              template<typename SUBNET>
              static void concat(tensor& out, const SUBNET& sub, size_t k_offset)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_output();
                  tt::copy_tensor(out, k_offset, t, 0, t.k());
                  k_offset += t.k();
                  concat_helper_impl<TAG_TYPES...>::concat(out, sub, k_offset);
              }
              template<typename SUBNET>
              static void split(const tensor& input, SUBNET& sub, size_t k_offset)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_gradient_input();
                  tt::copy_tensor(t, 0, input, k_offset, t.k());
                  k_offset += t.k();
                  concat_helper_impl<TAG_TYPES...>::split(input, sub, k_offset);
              }
          };
          template <template<typename> class TAG_TYPE>
          struct concat_helper_impl<TAG_TYPE>{
              constexpr static size_t tag_count() {return 1;}
              static void list_tags(std::ostream& out) 
              { 
                  out << tag_id<TAG_TYPE>::id;
              }
  
              template<typename SUBNET>
              static void resize_out(resizable_tensor& out, const SUBNET& sub, long sum_k)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_output();
                  out.set_size(t.num_samples(), t.k() + sum_k, t.nr(), t.nc());
              }
              template<typename SUBNET>
              static void concat(tensor& out, const SUBNET& sub, size_t k_offset)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_output();
                  tt::copy_tensor(out, k_offset, t, 0, t.k());
              }
              template<typename SUBNET>
              static void split(const tensor& input, SUBNET& sub, size_t k_offset)
              {
                  auto& t = layer<TAG_TYPE>(sub).get_gradient_input();
                  tt::copy_tensor(t, 0, input, k_offset, t.k());
              }
          };
      }
      // concat layer
      template<
          template<typename> class... TAG_TYPES
          >
      class concat_
      {
          static void list_tags(std::ostream& out) { impl::concat_helper_impl<TAG_TYPES...>::list_tags(out);};
  
      public:
          constexpr static size_t tag_count() {return impl::concat_helper_impl<TAG_TYPES...>::tag_count();};
  
          template <typename SUBNET>
          void setup (const SUBNET&)
          {
              // do nothing
          }
          template <typename SUBNET>
          void forward(const SUBNET& sub, resizable_tensor& output)
          {
              // the total depth of result is the sum of depths from all tags
              impl::concat_helper_impl<TAG_TYPES...>::resize_out(output, sub, 0);
  
              // copy output from each tag into different part result
              impl::concat_helper_impl<TAG_TYPES...>::concat(output, sub, 0);
          }
  
          template <typename SUBNET>
          void backward(const tensor& gradient_input, SUBNET& sub, tensor&)
          {
              // Gradient is split into parts for each tag layer
              impl::concat_helper_impl<TAG_TYPES...>::split(gradient_input, sub, 0);
          }
  
          point map_input_to_output(point p) const;
          point map_output_to_input(point p) const;
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const concat_& item, std::ostream& out)
          {
              serialize("concat_", out);
              size_t count = tag_count();
              serialize(count, out);
          }
  
          friend void deserialize(concat_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "concat_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::concat_.");
              size_t count_tags;
              deserialize(count_tags, in);
              if (count_tags != tag_count())
                  throw serialization_error("Invalid count of tags "+ std::to_string(count_tags) +", expecting " +
                                            std::to_string(tag_count()) +
                                                    " found while deserializing dlib::concat_.");
          }
  
          friend std::ostream& operator<<(std::ostream& out, const concat_& item)
          {
              out << "concat\t (";
              list_tags(out);
              out << ")";
              return out;
          }
  
          friend void to_xml(const concat_& item, std::ostream& out)
          {
              out << "<concat tags='";
              list_tags(out);
              out << "'/>\n";
          }
  
      private:
          resizable_tensor params; // unused
      };
  
  
      // concat layer definitions
      template <template<typename> class TAG1,
              template<typename> class TAG2,
              typename SUBNET>
      using concat2 = add_layer<concat_<TAG1, TAG2>, SUBNET>;
  
      template <template<typename> class TAG1,
              template<typename> class TAG2,
              template<typename> class TAG3,
              typename SUBNET>
      using concat3 = add_layer<concat_<TAG1, TAG2, TAG3>, SUBNET>;
  
      template <template<typename> class TAG1,
              template<typename> class TAG2,
              template<typename> class TAG3,
              template<typename> class TAG4,
              typename SUBNET>
      using concat4 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4>, SUBNET>;
  
      template <template<typename> class TAG1,
              template<typename> class TAG2,
              template<typename> class TAG3,
              template<typename> class TAG4,
              template<typename> class TAG5,
              typename SUBNET>
      using concat5 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4, TAG5>, SUBNET>;
  
      // inception layer will use tags internally. If user will use tags too, some conflicts
      // possible to exclude them, here are new tags specially for inceptions
      template <typename SUBNET> using itag0  = add_tag_layer< 1000 + 0, SUBNET>;
      template <typename SUBNET> using itag1  = add_tag_layer< 1000 + 1, SUBNET>;
      template <typename SUBNET> using itag2  = add_tag_layer< 1000 + 2, SUBNET>;
      template <typename SUBNET> using itag3  = add_tag_layer< 1000 + 3, SUBNET>;
      template <typename SUBNET> using itag4  = add_tag_layer< 1000 + 4, SUBNET>;
      template <typename SUBNET> using itag5  = add_tag_layer< 1000 + 5, SUBNET>;
      // skip to inception input
      template <typename SUBNET> using iskip  = add_skip_layer< itag0, SUBNET>;
  
      // here are some templates to be used for creating inception layer groups
      template <template<typename>class B1,
              template<typename>class B2,
              typename SUBNET>
      using inception2 = concat2<itag1, itag2, itag1<B1<iskip< itag2<B2< itag0<SUBNET>>>>>>>;
  
      template <template<typename>class B1,
              template<typename>class B2,
              template<typename>class B3,
              typename SUBNET>
      using inception3 = concat3<itag1, itag2, itag3, itag1<B1<iskip< itag2<B2<iskip< itag3<B3<  itag0<SUBNET>>>>>>>>>>;
  
      template <template<typename>class B1,
              template<typename>class B2,
              template<typename>class B3,
              template<typename>class B4,
              typename SUBNET>
      using inception4 = concat4<itag1, itag2, itag3, itag4,
                  itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip<  itag4<B4<  itag0<SUBNET>>>>>>>>>>>>>;
  
      template <template<typename>class B1,
              template<typename>class B2,
              template<typename>class B3,
              template<typename>class B4,
              template<typename>class B5,
              typename SUBNET>
      using inception5 = concat5<itag1, itag2, itag3, itag4, itag5,
                  itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip<  itag4<B4<iskip<  itag5<B5<  itag0<SUBNET>>>>>>>>>>>>>>>>;
  
  // ----------------------------------------------------------------------------------------
  // ----------------------------------------------------------------------------------------
  
      const double DEFAULT_L2_NORM_EPS = 1e-5;
  
      class l2normalize_
      {
      public:
          explicit l2normalize_(
              double eps_ = DEFAULT_L2_NORM_EPS
          ) : 
              eps(eps_)
          {
          }
  
          double get_eps() const { return eps; }
  
          template <typename SUBNET>
          void setup (const SUBNET& /*sub*/)
          {
          }
  
          void forward_inplace(const tensor& input, tensor& output)
          {
              tt::inverse_norms(norm, input, eps);
              tt::scale_rows(output, input, norm);
          } 
  
          void backward_inplace(
              const tensor& computed_output, 
              const tensor& gradient_input, 
              tensor& data_grad, 
              tensor& /*params_grad*/
          )
          {
              if (is_same_object(gradient_input, data_grad))
              {
                  tt::dot_prods(temp, gradient_input, computed_output);
                  tt::scale_rows2(0, data_grad, gradient_input, computed_output, temp, norm);
              }
              else
              {
                  tt::dot_prods(temp, gradient_input, computed_output);
                  tt::scale_rows2(1, data_grad, gradient_input, computed_output, temp, norm);
              }
          }
  
          const tensor& get_layer_params() const { return params; }
          tensor& get_layer_params() { return params; }
  
          friend void serialize(const l2normalize_& item, std::ostream& out)
          {
              serialize("l2normalize_", out);
              serialize(item.eps, out);
          }
  
          friend void deserialize(l2normalize_& item, std::istream& in)
          {
              std::string version;
              deserialize(version, in);
              if (version != "l2normalize_")
                  throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::l2normalize_.");
              deserialize(item.eps, in);
          }
  
          friend std::ostream& operator<<(std::ostream& out, const l2normalize_& item)
          {
              out << "l2normalize";
              out << " eps="<<item.eps;
              return out;
          }
  
          friend void to_xml(const l2normalize_& item, std::ostream& out)
          {
              out << "<l2normalize";
              out << " eps='"<<item.eps<<"'";
              out << "/>\n";
          }
      private:
          double eps;
  
          resizable_tensor params; // unused
          // Here only to avoid reallocation and as a cache between forward/backward
          // functions.  
          resizable_tensor norm;
          resizable_tensor temp;
      };
  
      template <typename SUBNET>
      using l2normalize = add_layer<l2normalize_, SUBNET>;
  
  // ----------------------------------------------------------------------------------------
  
  }
  
  #endif // DLIB_DNn_LAYERS_H_
  
  
  