text: /*
      Copyright (c) 2013, Taiga Nomi
      All rights reserved.
      
      Redistribution and use in source and binary forms, with or without
      modification, are permitted provided that the following conditions are met:
      * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
      * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
      * Neither the name of the <organization> nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
  
      THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 
      EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
      WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
      DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 
      DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
      (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
      LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 
      ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
      (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
      SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
  #include <iostream>
  #include <memory>
  #include "tiny_cnn/tiny_cnn.h"
  
  using namespace tiny_cnn;
  using namespace tiny_cnn::activation;
  using namespace std;
  
  void sample1_convnet(const string& data_dir_path = "../../data");
  void sample2_mlp();
  void sample3_dae();
  void sample4_dropout();
  void sample5_unbalanced_training_data(const string& data_dir_path = "../../data");
  
  int main(int argc, char** argv) {
      try {
          if (argc == 2) {
              sample1_convnet(argv[1]);
          } else {
              sample1_convnet();
          }
      }
      catch (const nn_error& e)
      {
          std::cout << e.what() << std::endl;
      }
  }
  
  ///////////////////////////////////////////////////////////////////////////////
  // learning convolutional neural networks (LeNet-5 like architecture)
  void sample1_convnet(const string& data_dir_path) {
      // construct LeNet-5 architecture
      network<mse, gradient_descent_levenberg_marquardt> nn;
  
      // connection table [Y.Lecun, 1998 Table.1]
  #define O true
  #define X false
      static const bool connection [] = {
          O, X, X, X, O, O, O, X, X, O, O, O, O, X, O, O,
          O, O, X, X, X, O, O, O, X, X, O, O, O, O, X, O,
          O, O, O, X, X, X, O, O, O, X, X, O, X, O, O, O,
          X, O, O, O, X, X, O, O, O, O, X, X, O, X, O, O,
          X, X, O, O, O, X, X, O, O, O, O, X, O, O, X, O,
          X, X, X, O, O, O, X, X, O, O, O, O, X, O, O, O
      };
  #undef O
  #undef X
  
      nn << convolutional_layer<tan_h>(32, 32, 5, 1, 6) // 32x32 in, 5x5 kernel, 1-6 fmaps conv
         << average_pooling_layer<tan_h>(28, 28, 6, 2) // 28x28 in, 6 fmaps, 2x2 subsampling
         << convolutional_layer<tan_h>(14, 14, 5, 6, 16,
                                       connection_table(connection, 6, 16)) // with connection-table
         << average_pooling_layer<tan_h>(10, 10, 16, 2)
         << convolutional_layer<tan_h>(5, 5, 5, 16, 120)
         << fully_connected_layer<tan_h>(120, 10);
  
      std::cout << "load models..." << std::endl;
  
      // load MNIST dataset
      std::vector<label_t> train_labels, test_labels;
      std::vector<vec_t> train_images, test_images;
  
      parse_mnist_labels(data_dir_path + "/train-labels.idx1-ubyte", &train_labels);
      parse_mnist_images(data_dir_path + "/train-images.idx3-ubyte", &train_images, -1.0, 1.0, 2, 2);
      parse_mnist_labels(data_dir_path + "/t10k-labels.idx1-ubyte", &test_labels);
      parse_mnist_images(data_dir_path + "/t10k-images.idx3-ubyte", &test_images, -1.0, 1.0, 2, 2);
  
      std::cout << "start learning" << std::endl;
  
      progress_display disp(train_images.size());
      timer t;
      int minibatch_size = 10;
  
      nn.optimizer().alpha *= std::sqrt(minibatch_size);
  
      // create callback
      auto on_enumerate_epoch = [&](){
          std::cout << t.elapsed() << "s elapsed." << std::endl;
  
          tiny_cnn::result res = nn.test(test_images, test_labels);
  
          std::cout << nn.optimizer().alpha << "," << res.num_success << "/" << res.num_total << std::endl;
  
          nn.optimizer().alpha *= 0.85; // decay learning rate
          nn.optimizer().alpha = std::max((tiny_cnn::float_t)0.00001, nn.optimizer().alpha);
  
          disp.restart(train_images.size());
          t.restart();
      };
  
      auto on_enumerate_minibatch = [&](){ 
          disp += minibatch_size; 
      };
      
      // training
      nn.train(train_images, train_labels, minibatch_size, 20, on_enumerate_minibatch, on_enumerate_epoch);
  
      std::cout << "end training." << std::endl;
  
      // test and show results
      nn.test(test_images, test_labels).print_detail(std::cout);
  
      // save networks
      std::ofstream ofs("LeNet-weights");
      ofs << nn;
  }
  
  
  ///////////////////////////////////////////////////////////////////////////////
  // learning 3-Layer Networks
  void sample2_mlp()
  {
      const cnn_size_t num_hidden_units = 500;
  
  #if defined(_MSC_VER) && _MSC_VER < 1800
      // initializer-list is not supported
      int num_units[] = { 28 * 28, num_hidden_units, 10 };
      auto nn = make_mlp<mse, gradient_descent, tan_h>(num_units, num_units + 3);
  #else
      auto nn = make_mlp<mse, gradient_descent, tan_h>({ 28 * 28, num_hidden_units, 10 });
  #endif
  
      // load MNIST dataset
      std::vector<label_t> train_labels, test_labels;
      std::vector<vec_t> train_images, test_images;
  
      parse_mnist_labels("../../data/train-labels.idx1-ubyte", &train_labels);
      parse_mnist_images("../../data/train-images.idx3-ubyte", &train_images, -1.0, 1.0, 0, 0);
      parse_mnist_labels("../../data/t10k-labels.idx1-ubyte", &test_labels);
      parse_mnist_images("../../data/t10k-images.idx3-ubyte", &test_images, -1.0, 1.0, 0, 0);
  
      nn.optimizer().alpha = 0.001;
      
      progress_display disp(train_images.size());
      timer t;
  
      // create callback
      auto on_enumerate_epoch = [&](){
          std::cout << t.elapsed() << "s elapsed." << std::endl;
  
          tiny_cnn::result res = nn.test(test_images, test_labels);
  
          std::cout << nn.optimizer().alpha << "," << res.num_success << "/" << res.num_total << std::endl;
  
          nn.optimizer().alpha *= 0.85; // decay learning rate
          nn.optimizer().alpha = std::max((tiny_cnn::float_t)0.00001, nn.optimizer().alpha);
  
          disp.restart(train_images.size());
          t.restart();
      };
  
      auto on_enumerate_data = [&](){ 
          ++disp; 
      };  
  
      nn.train(train_images, train_labels, 1, 20, on_enumerate_data, on_enumerate_epoch);
  }
  
  ///////////////////////////////////////////////////////////////////////////////
  // denoising auto-encoder
  void sample3_dae()
  {
  #if defined(_MSC_VER) && _MSC_VER < 1800
      // initializer-list is not supported
      int num_units[] = { 100, 400, 100 };
      auto nn = make_mlp<mse, gradient_descent, tan_h>(num_units, num_units + 3);
  #else
      auto nn = make_mlp<mse, gradient_descent, tan_h>({ 100, 400, 100 });
  #endif
  
      std::vector<vec_t> train_data_original;
  
      // load train-data
  
      std::vector<vec_t> train_data_corrupted(train_data_original);
  
      for (auto& d : train_data_corrupted) {
          d = corrupt(move(d), 0.1, 0.0); // corrupt 10% data
      }
  
      // learning 100-400-100 denoising auto-encoder
      nn.train(train_data_corrupted, train_data_original);
  }
  
  ///////////////////////////////////////////////////////////////////////////////
  // dropout-learning
  
  void sample4_dropout()
  {
      typedef network<mse, gradient_descent> Network;
      Network nn;
      cnn_size_t input_dim    = 28*28;
      cnn_size_t hidden_units = 800;
      cnn_size_t output_dim   = 10;
  
      fully_connected_layer<tan_h> f1(input_dim, hidden_units);
      dropout_layer dropout(hidden_units, 0.5);
      fully_connected_layer<tan_h> f2(hidden_units, output_dim);
      nn << f1 << dropout << f2;
  
      nn.optimizer().alpha = 0.003; // TODO: not optimized
      nn.optimizer().lambda = 0.0;
  
      // load MNIST dataset
      std::vector<label_t> train_labels, test_labels;
      std::vector<vec_t> train_images, test_images;
  
      parse_mnist_labels("../../data/train-labels.idx1-ubyte", &train_labels);
      parse_mnist_images("../../data/train-images.idx3-ubyte", &train_images, -1.0, 1.0, 0, 0);
      parse_mnist_labels("../../data/t10k-labels.idx1-ubyte", &test_labels);
      parse_mnist_images("../../data/t10k-images.idx3-ubyte", &test_images, -1.0, 1.0, 0, 0);
  
      // load train-data, label_data
      progress_display disp(train_images.size());
      timer t;
  
      // create callback
      auto on_enumerate_epoch = [&](){
          std::cout << t.elapsed() << "s elapsed." << std::endl;
    
          dropout.set_context(net_phase::test);
          tiny_cnn::result res = nn.test(test_images, test_labels);
          dropout.set_context(net_phase::train);
  
  
          std::cout << nn.optimizer().alpha << "," << res.num_success << "/" << res.num_total << std::endl;
  
          nn.optimizer().alpha *= 0.99; // decay learning rate
          nn.optimizer().alpha = std::max((tiny_cnn::float_t)0.00001, nn.optimizer().alpha);
  
          disp.restart(train_images.size());
          t.restart();
      };
  
      auto on_enumerate_data = [&](){
          ++disp;
      };
  
      nn.train(train_images, train_labels, 1, 100, on_enumerate_data, on_enumerate_epoch);
  
      // change context to enable all hidden-units
      //f1.set_context(dropout::test_phase);
      //std::cout << res.num_success << "/" << res.num_total << std::endl;
  }
  
  #include "tiny_cnn/util/target_cost.h"
  
  ///////////////////////////////////////////////////////////////////////////////
  // learning unbalanced training data
  
  void sample5_unbalanced_training_data(const string& data_dir_path)
  {
      const cnn_size_t num_hidden_units = 20; // keep the network relatively simple
  
  #if defined(_MSC_VER) && _MSC_VER < 1800
      // initializer-list is not supported
      int num_units[] = { 28 * 28, num_hidden_units, 10 };
      auto nn_standard = make_mlp<mse, gradient_descent, tan_h>(num_units, num_units + 3);
      auto nn_balanced = make_mlp<mse, gradient_descent, tan_h>(num_units, num_units + 3);
  #else
      auto nn_standard = make_mlp<mse, gradient_descent, tan_h>({ 28 * 28, num_hidden_units, 10 });
      auto nn_balanced = make_mlp<mse, gradient_descent, tan_h>({ 28 * 28, num_hidden_units, 10 });
  #endif
  
      // load MNIST dataset
      std::vector<label_t> train_labels, test_labels;
      std::vector<vec_t>   train_images, test_images;
  
      parse_mnist_labels(data_dir_path + "/train-labels.idx1-ubyte", &train_labels);
      parse_mnist_images(data_dir_path + "/train-images.idx3-ubyte", &train_images, -1.0, 1.0, 0, 0);
      parse_mnist_labels(data_dir_path + "/t10k-labels.idx1-ubyte", &test_labels);
      parse_mnist_images(data_dir_path + "/t10k-images.idx3-ubyte", &test_images, -1.0, 1.0, 0, 0);
  
      { // create an unbalanced training set
          std::vector<label_t> train_labels_unbalanced;
          std::vector<vec_t>   train_images_unbalanced;
          train_labels_unbalanced.reserve(train_labels.size());
          train_images_unbalanced.reserve(train_images.size());
  
          for (size_t i = 0, end = train_labels.size(); i < end; ++i) {
              const label_t label = train_labels[i];
  
              // drop most 0s, 1s, 2s, 3s, and 4s
              const bool keep_sample = label >= 5 || bernoulli(0.005);
  
              if (keep_sample) {
                  train_labels_unbalanced.push_back(label);
                  train_images_unbalanced.push_back(train_images[i]);
              }
          }
  
          // keep the newly created unbalanced training set
          std::swap(train_labels, train_labels_unbalanced);
          std::swap(train_images, train_images_unbalanced);
      }
  
      nn_standard.optimizer().alpha = 0.001;
      nn_balanced.optimizer().alpha = 0.001;
  
      progress_display disp(train_images.size());
      timer t;
  
      const int minibatch_size = 10;
  
      auto nn = &nn_standard; // the network referred to by the callbacks
  
      // create callbacks - as usual
      auto on_enumerate_epoch = [&](){
          std::cout << t.elapsed() << "s elapsed." << std::endl;
  
          tiny_cnn::result res = nn->test(test_images, test_labels);
  
          std::cout << nn->optimizer().alpha << "," << res.num_success << "/" << res.num_total << std::endl;
  
          nn->optimizer().alpha *= 0.85; // decay learning rate
          nn->optimizer().alpha = std::max((tiny_cnn::float_t)0.00001, nn->optimizer().alpha);
  
          disp.restart(train_images.size());
          t.restart();
      };
  
      auto on_enumerate_data = [&](){
          disp += minibatch_size;
      };
  
      // first train the standard network (default cost - equal for each sample)
      // - note that it does not learn the classes 0-4
      nn_standard.train(train_images, train_labels, minibatch_size, 20, on_enumerate_data, on_enumerate_epoch, true, CNN_TASK_SIZE);
  
      // then train another network, now with explicitly supplied target costs (aim: a more balanced predictor)
      // - note that it can learn the classes 0-4 (at least somehow)
      nn = &nn_balanced;
      const auto target_cost = create_balanced_target_cost(train_labels, 0.8);
      nn_balanced.train(train_images, train_labels, minibatch_size, 20, on_enumerate_data, on_enumerate_epoch, true, CNN_TASK_SIZE, &target_cost);
  
      // test and show results
      std::cout << std::endl << "Standard training (implicitly assumed equal cost for every sample):" << std::endl;
      nn_standard.test(test_images, test_labels).print_detail(std::cout);
  
      std::cout << std::endl << "Balanced training (explicitly supplied target costs):" << std::endl;
      nn_balanced.test(test_images, test_labels).print_detail(std::cout);
  }
  