text: # Code adapted from https://github.com/Newmu/Theano-Tutorials
  import sys, time, os
  from ntpath import basename
  from os.path import splitext
  from itertools import product
  import cPickle as pickle
  import theano
  from theano import tensor as T
  import numpy as np
  from sklearn.cross_validation import KFold
  from params import nnet_params_dict, feats_train_folder
  
  def set_trace():
      from IPython.core.debugger import Pdb
      import sys
      Pdb(color_scheme='Linux').set_trace(sys._getframe().f_back)
  
  def floatX(X):
      return np.asarray(X, dtype=theano.config.floatX)
  
  def zerosX(X):
  	return np.zeros(X, dtype=theano.config.floatX)
  
  def init_weights(shape):
      return theano.shared(floatX(np.random.randn(*shape) * 0.01))
  
  def sgd(cost, params, gamma):
      grads = T.grad(cost=cost, wrt=params)
      updates = []
      for p, g in zip(params, grads):
          updates.append([p, p - g * gamma])
      return updates
  
  def model(X, w_h, w_o):
      h = T.nnet.sigmoid(T.dot(X, w_h))
      pyx = T.nnet.softmax(T.dot(h, w_o))
      return pyx
  
  verbose = True
  # train on every perturbed dataset
  filepaths = np.loadtxt("include_data.csv", dtype=object, delimiter=",")
  for (include, train_filename, test_filename) in filepaths:
      if include == '1':
          print '\nExecuting {}'.format(train_filename)
          # Load training and test sets
          x_train = np.load(os.path.join(feats_train_folder, train_filename)).astype(np.float32)
          y_train = (np.eye(2, dtype=np.float32)[x_train[:,-1].astype(int)])
  
          # remove label column from x_train
          x_train = x_train[:,:-1]
  
          # Network topology
          n_inputs = x_train.shape[1]
          n_outputs = len(np.unique(y_train))
  
          # Cross-validation and Neural Net parameters
          n_folds = nnet_params_dict['n_folds']
          alphas = nnet_params_dict['alphas']
          gammas = nnet_params_dict['gammas']
          batch_sizes = nnet_params_dict['batch_sizes']
          max_epoch = nnet_params_dict['max_epoch']
  
          # Dictionary to store results
          results_dict = {}
  
          params_mat = np.array([x for x in product(alphas, gammas, batch_sizes)], 
  			       dtype=theano.config.floatX)
          params_mat = np.column_stack((params_mat,
                                           zerosX(params_mat.shape[0]),
                                           zerosX(params_mat.shape[0]),
                                           zerosX(params_mat.shape[0])))
  
          for param_idx in xrange(params_mat.shape[0]):
              alpha = params_mat[param_idx, 0]
              gamma = params_mat[param_idx, 1]
              batch_size = int(params_mat[param_idx, 2])
              n_hidden = int((x_train.shape[0] / n_folds)/(alpha*(n_inputs+n_outputs)))
  
              # Initialize weights
              w_h = init_weights((n_inputs, n_hidden))
              w_o = init_weights((n_hidden, n_outputs))
  
              # Initialize NN classifier
              X = T.fmatrix()
              Y = T.fmatrix()
  
              py_x = model(X, w_h, w_o)
              y_x = T.argmax(py_x, axis=1)
  
              cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y), 
  			  dtype=theano.config.floatX)
  
              params = [w_h, w_o]
  
              updates = sgd(cost, params, gamma=gamma)
  
              train = theano.function(inputs=[X, Y],
                                      outputs=cost,
                                      updates=updates,
                                      allow_input_downcast=True)
  
              predict = theano.function(inputs=[X],
                                        outputs=y_x,
                                        allow_input_downcast=True)
  
  
              # Test on validation set
              model_str = 'alpha {} gamma {} batch size {}'.format(alpha,
                                                                  gamma,
                                                                  batch_size)
              print model_str
  
              kf = KFold(x_train.shape[0], n_folds=n_folds)
              error_rates = []
              test_costs = []
              running_time = []
  
              fold = 1
              for train_idx, val_idx in kf:
                  start_time = time.time()
                  for i in range(max_epoch):
                      for start, end in zip(range(0, len(x_train[train_idx]),
                                            batch_size),
                                            range(batch_size, len(x_train[train_idx]),
                                            batch_size)):
                          test_cost = train(x_train[train_idx][start:end],
                                            y_train[train_idx][start:end])
  
                      error_rate = 1 - np.mean(np.argmax(y_train[val_idx], axis=1) ==
                                                         predict(x_train[val_idx]))
                      if (i % (max_epoch / 4)) == 0 and verbose:
                          print 'fold {}, epoch {}, error rate {}, cost {}'.format(fold, i+1,
                                                                      error_rate,
                                                                      test_cost)
                  error_rates.append(error_rate)
                  test_costs.append(test_cost)
                  running_time.append(np.around((time.time() - start_time) / 60., 1))
                  fold += 1
  
              params_mat[param_idx, 3] = np.mean(error_rate)
              params_mat[param_idx, 4] = np.mean(test_cost)
              params_mat[param_idx, 5] = np.mean(running_time)
  
              print 'alpha {} gamma {} batchsize {} error rate {} test cost {} running time {}'.format(params_mat[param_idx,0],
                  params_mat[param_idx,1],
                  params_mat[param_idx,2],
                  params_mat[param_idx,3],
                  params_mat[param_idx,4],
                  params_mat[param_idx,5])
  
  
          # Save params matrix to disk
          params_mat.dump('results/train/{}_results.np'.format(train_filename[:-3]))
  