title: tensorflow DNN with two hidden layers significantly slower and produce much worse result than using scikit flow
body: <p>my bio dataset has 20K rows and 170 feature. I'm doing dnn regression to predict bioactivity. ( one unit output layer with linear equation and two hidden layers).  It ran very slow on my cpu and produced really bad r-square (negative).  Then I ran it with skflow with the same network architecture. It was way way faster (more than 100x) and I got r2 much better than the previous run (r2=0.3), although not a great result. Does anyone know why? anything wrong with my code? what is the difference between my code and  underlying skflow code?  is my loss function defined correctly?
  Help is very much appreciated. 
  Below are the codes:</p>
  
  <pre><code># with scikit flow 
  dnn_reg = skflow.TensorFlowDNNRegressor(hidden_units=[200,500], steps=3000, learning_rate=0.5)
  dnn_reg.fit(x_train, y_train)
  pred_train = dnn_reg.predict (x_train)
  pred_valid = dnn_reg.predict (x_valid)
  print ('r-square for training set', r2_score(y_train, pred_train))
  print ('r-square for validation set',r2_score(y_valid, pred_valid))
  
  # tensorflow code
  
  n_samples = 15000
  n_features = 171
  batch_size = 1000
  num_batch = n_samples/batch_size
  hidden1 = 200
  hidden2 = 100
  learning_rate=0.01
  n_epoch=3000
  
  graph = tf.Graph()
  with graph.as_default():
      #constant and palceholder    
      tf_train_data = tf.placeholder(tf.float32, shape=(batch_size, n_features))
      tf_train_act = tf.placeholder(tf.float32, shape=(batch_size))
      tf_valid_data=tf.constant (x_valid.astype(np.float32))
  
  
      # variables
      w1 = tf.Variable(tf.truncated_normal([n_features, hidden1]), name='weight1')
      b1 = tf.Variable(tf.zeros([hidden1]), name='bias1')
      w2 = tf.Variable(tf.truncated_normal([hidden1, hidden2]), name='weight2')
      b2 = tf.Variable(tf.zeros([hidden2]), name='bias2')
      w3 = tf.Variable(tf.truncated_normal([hidden2, 1]), name='weight3')
      b3 = tf.Variable(tf.zeros([1]), name='bias3')
  
      #parameter histogram    
      w1_hist = tf.histogram_summary('weight_input', w1)
      w2_hist = tf.histogram_summary('weight2', w2)
      w3_hist = tf.histogram_summary('weight3', w3)
      b1_hist = tf.histogram_summary('bias1', b1)
      b2_hist = tf.histogram_summary('bias2', b2)
      b3_hist = tf.histogram_summary('bias3', b3)
      #y_hist = tf.histogram_summary('y', y_train)   
  
      #training computation
      def forward_prop (input):
          with tf.name_scope('hidden_1') as scope:
              h1 = tf.nn.relu(tf.matmul(input, w1)+b1)
          with tf.name_scope('hidden_2') as scope:
              h2 = tf.nn.relu(tf.matmul(h1, w2)+b2)
          with tf.name_scope('output') as scope: 
              output = tf.matmul(h2, w3)+b3
          return (output)
  
      #forward propagation
      output = forward_prop(tf_train_data)
      with tf.name_scope('cost') as scope:
  
          loss=tf.sqrt(tf.reduce_mean(tf.square(tf.sub(tf_train_act, output))))
          cost_summary = tf.scalar_summary('cost', loss)
  
      #optimizer
      with tf.name_scope('train') as scope: 
          optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(loss)
  
      #predictions
          train_prediction = output
          valid_prediction = forward_prop(tf_valid_data)
  
  
  
  with tf.Session(graph=graph) as session:
  
      session.run(tf.initialize_all_variables())
      print ('initialized')
  
      merged = tf.merge_all_summaries()
      writer = tf.train.SummaryWriter ('./logs/log1', session.graph)
  
      for epoch in range(n_epoch):
          mini = np.array_split(range(y_train.shape[0]), num_batch)
          for idx in mini[:-1]:
              batch_x = x_train[idx]
              batch_y = y_train[idx]
              feed_dict = {tf_train_data:batch_x, tf_train_act:batch_y}
              _,l, pred_train = session.run([optimizer, loss, output], feed_dict=feed_dict)
  
          if epoch % 100 == 0:
              print ('minibatch loss at step %d: %f' % (epoch, l))
              print ('minibatch r2: %0.1f' % r2_score(batch_y, pred_train))
              print ('validation r2: %0.1f' % r2_score(y_valid, valid_prediction.eval()))
  </code></pre>