mnist_train.py
def main():
  """ Train an RNN for sequential (possibly permuted) MNIST recognition. """

  args, params_str, layer_kwargs = parse_args()

  save_dir = os.path.join(args.data_dir, 'results', params_str)
  if os.path.exists(save_dir):
    shutil.rmtree(save_dir)
  os.makedirs(save_dir)

  outs = mnist.load_split(args.data_dir, val=True, permute=args.permute, normalize=True, num_val=2000, seed=0)
  train_images, train_labels, val_images, val_labels = outs
  if args.debug:
    train_images = train_images[:, ::4, ::4, :]
    val_images = val_images[:, ::4, ::4, :]

  # Flatten the images.
  train_inputs = train_images.reshape([len(train_images), -1, INPUT_SIZE])
  val_inputs = val_images.reshape([len(val_images), -1, INPUT_SIZE])

  # Align sequence-level labels with the appropriate time steps by padding with NaNs,
  # and to do so, first convert the labels to floats.
  length = train_inputs.shape[1]
  pad = lambda x: np.pad(x, [[0, 0], [length - 1, 0], [0, 0]], mode='constant', constant_values=np.nan)
  train_labels = pad(train_labels.reshape([-1, 1, 1]).astype(np.float))
  val_labels = pad(val_labels.reshape([-1, 1, 1]).astype(np.float))

  train_batches = utils.full_bptt_batch_generator(train_inputs, train_labels, BATCH_SIZE)

  model = models.RNNClassificationModel(args.layer_type, INPUT_SIZE, TARGET_SIZE, args.num_hidden_units,
                                        args.activation_type, **layer_kwargs)

  Optimizer = getattr(optimizers, args.optimizer)
  optimize_op = Optimizer(args.learning_rate).minimize(model.valid_seqwise_loss_for_opt)

  def _error_rate(valid_predictions, valid_targets):
    incorrect_mask = tf.logical_not(tf.equal(tf.argmax(valid_predictions, 1), tf.argmax(valid_targets, 1)))
    return tf.reduce_mean(tf.to_float(incorrect_mask))
  model.error_rate = _error_rate(model.valid_predictions, model.valid_targets)

  tf.summary.scalar('train loss', model.valid_seqwise_loss, collections=['train'])
  tf.summary.scalar('train error rate', model.error_rate, collections=['train'])

  model.val_error_rate = tf.placeholder(tf.float32, name='val_error_rate')
  tf.summary.scalar('val error rate', model.val_error_rate, collections=['val'])

  train_summary_op = tf.summary.merge_all('train')
  val_summary_op = tf.summary.merge_all('val')

  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  sess = tf.Session(config=config)
  sess.run(tf.global_variables_initializer())

  file_writer = tf.summary.FileWriter(save_dir, graph=sess.graph, flush_secs=10)
  saver = tf.train.Saver()

  best_val_error_rate = 1.0
  start_time = time.time()
  for step in range(NUM_OPT_STEPS):

    batch_inputs, batch_labels = next(train_batches)
    batch_targets = utils.one_hot(np.squeeze(batch_labels, 2), TARGET_SIZE)

    sess.run(optimize_op,
             feed_dict={model.inputs: batch_inputs,
                        model.targets: batch_targets})

    if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0:

      error_rate, summary = sess.run([model.error_rate, train_summary_op],
                                     feed_dict={model.inputs: batch_inputs,
                                                model.targets: batch_targets})

      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'train_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f' % (params_str, time.time() - start_time, step, error_rate)
        print(line, file=f)

    if step % NUM_STEPS_PER_VAL_SUMMARY == 0:

      val_batches = utils.full_bptt_batch_generator(val_inputs, val_labels, VAL_BATCH_SIZE, num_epochs=1,
                                                    shuffle=False)
      error_rates = []
      for i, (batch_inputs, batch_labels) in enumerate(val_batches):
        batch_targets = utils.one_hot(np.squeeze(batch_labels, 2), TARGET_SIZE)
        valid_predictions, valid_targets, batch_error_rates = sess.run(
          [model.valid_predictions, model.valid_targets, model.error_rate],
          feed_dict={model.inputs: batch_inputs,
                     model.targets: batch_targets}
        )
        error_rates.append(batch_error_rates)
        if args.debug and i == 0:
          num_samples = 25
          print('Step: %d. Some targets and predictions:' % step)
          print(np.argmax(valid_targets[:num_samples], axis=1))
          print(np.argmax(valid_predictions[:num_samples], axis=1))

      # This is approximate if the validation batch size doesn't evenly divide
      # the number of validation examples.
      val_error_rate = np.mean(error_rates, dtype=np.float)
      if val_error_rate < best_val_error_rate:
        best_val_error_rate = val_error_rate
        saver.save(sess, os.path.join(save_dir, 'model.ckpt'))

      summary = sess.run(val_summary_op, feed_dict={model.val_error_rate: val_error_rate})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'val_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f %.4f' % (params_str, time.time() - start_time, step,
                                           val_error_rate, best_val_error_rate)
        print(line, file=f)
        if args.debug:
          print(line)

  file_writer.close()
 Train an RNN for sequential (possibly permuted) MNIST recognition. 
Train an RNN for sequential (possibly permuted) MNIST recognition.
Train an RNN for sequential ( possibly permuted ) MNIST recognition .
def main():
  args, params_str, layer_kwargs = parse_args()
  save_dir = os.path.join(args.data_dir, 'results', params_str)
  if os.path.exists(save_dir):
    shutil.rmtree(save_dir)
  os.makedirs(save_dir)
  outs = mnist.load_split(args.data_dir, val=True, permute=args.permute, normalize=True, num_val=2000, seed=0)
  train_images, train_labels, val_images, val_labels = outs
  if args.debug:
    train_images = train_images[:, ::4, ::4, :]
    val_images = val_images[:, ::4, ::4, :]
  train_inputs = train_images.reshape([len(train_images), -1, INPUT_SIZE])
  val_inputs = val_images.reshape([len(val_images), -1, INPUT_SIZE])
  length = train_inputs.shape[1]
  pad = lambda x: np.pad(x, [[0, 0], [length - 1, 0], [0, 0]], mode='constant', constant_values=np.nan)
  train_labels = pad(train_labels.reshape([-1, 1, 1]).astype(np.float))
  val_labels = pad(val_labels.reshape([-1, 1, 1]).astype(np.float))
  train_batches = utils.full_bptt_batch_generator(train_inputs, train_labels, BATCH_SIZE)
  model = models.RNNClassificationModel(args.layer_type, INPUT_SIZE, TARGET_SIZE, args.num_hidden_units,
                                        args.activation_type, **layer_kwargs)
  Optimizer = getattr(optimizers, args.optimizer)
  optimize_op = Optimizer(args.learning_rate).minimize(model.valid_seqwise_loss_for_opt)
  def _error_rate(valid_predictions, valid_targets):
    incorrect_mask = tf.logical_not(tf.equal(tf.argmax(valid_predictions, 1), tf.argmax(valid_targets, 1)))
    return tf.reduce_mean(tf.to_float(incorrect_mask))
  model.error_rate = _error_rate(model.valid_predictions, model.valid_targets)
  tf.summary.scalar('train loss', model.valid_seqwise_loss, collections=['train'])
  tf.summary.scalar('train error rate', model.error_rate, collections=['train'])
  model.val_error_rate = tf.placeholder(tf.float32, name='val_error_rate')
  tf.summary.scalar('val error rate', model.val_error_rate, collections=['val'])
  train_summary_op = tf.summary.merge_all('train')
  val_summary_op = tf.summary.merge_all('val')
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True
  sess = tf.Session(config=config)
  sess.run(tf.global_variables_initializer())
  file_writer = tf.summary.FileWriter(save_dir, graph=sess.graph, flush_secs=10)
  saver = tf.train.Saver()
  best_val_error_rate = 1.0
  start_time = time.time()
  for step in range(NUM_OPT_STEPS):
    batch_inputs, batch_labels = next(train_batches)
    batch_targets = utils.one_hot(np.squeeze(batch_labels, 2), TARGET_SIZE)
    sess.run(optimize_op,
             feed_dict={model.inputs: batch_inputs,
                        model.targets: batch_targets})
    if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0:
      error_rate, summary = sess.run([model.error_rate, train_summary_op],
                                     feed_dict={model.inputs: batch_inputs,
                                                model.targets: batch_targets})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'train_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f' % (params_str, time.time() - start_time, step, error_rate)
        print(line, file=f)
    if step % NUM_STEPS_PER_VAL_SUMMARY == 0:
      val_batches = utils.full_bptt_batch_generator(val_inputs, val_labels, VAL_BATCH_SIZE, num_epochs=1,
                                                    shuffle=False)
      error_rates = []
      for i, (batch_inputs, batch_labels) in enumerate(val_batches):
        batch_targets = utils.one_hot(np.squeeze(batch_labels, 2), TARGET_SIZE)
        valid_predictions, valid_targets, batch_error_rates = sess.run(
          [model.valid_predictions, model.valid_targets, model.error_rate],
          feed_dict={model.inputs: batch_inputs,
                     model.targets: batch_targets}
        )
        error_rates.append(batch_error_rates)
        if args.debug and i == 0:
          num_samples = 25
          print('Step: %d. Some targets and predictions:' % step)
          print(np.argmax(valid_targets[:num_samples], axis=1))
          print(np.argmax(valid_predictions[:num_samples], axis=1))
      val_error_rate = np.mean(error_rates, dtype=np.float)
      if val_error_rate < best_val_error_rate:
        best_val_error_rate = val_error_rate
        saver.save(sess, os.path.join(save_dir, 'model.ckpt'))
      summary = sess.run(val_summary_op, feed_dict={model.val_error_rate: val_error_rate})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'val_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f %.4f' % (params_str, time.time() - start_time, step,
                                           val_error_rate, best_val_error_rate)
        print(line, file=f)
        if args.debug:
          print(line)
  file_writer.close()
def main ( ) : args , params_str , layer_kwargs = parse_args ( ) save_dir = os . path . join ( args . data_dir , 'results' , params_str ) if os . path . exists ( save_dir ) : shutil . rmtree ( save_dir ) os . makedirs ( save_dir ) outs = mnist . load_split ( args . data_dir , val = True , permute = args . permute , normalize = True , num_val = 2000 , seed = 0 ) train_images , train_labels , val_images , val_labels = outs if args . debug : train_images = train_images [ : , : : 4 , : : 4 , : ] val_images = val_images [ : , : : 4 , : : 4 , : ] train_inputs = train_images . reshape ( [ len ( train_images ) , - 1 , INPUT_SIZE ] ) val_inputs = val_images . reshape ( [ len ( val_images ) , - 1 , INPUT_SIZE ] ) length = train_inputs . shape [ 1 ] pad = lambda x : np . pad ( x , [ [ 0 , 0 ] , [ length - 1 , 0 ] , [ 0 , 0 ] ] , mode = 'constant' , constant_values = np . nan ) train_labels = pad ( train_labels . reshape ( [ - 1 , 1 , 1 ] ) . astype ( np . float ) ) val_labels = pad ( val_labels . reshape ( [ - 1 , 1 , 1 ] ) . astype ( np . float ) ) train_batches = utils . full_bptt_batch_generator ( train_inputs , train_labels , BATCH_SIZE ) model = models . RNNClassificationModel ( args . layer_type , INPUT_SIZE , TARGET_SIZE , args . num_hidden_units , args . activation_type , ** layer_kwargs ) Optimizer = getattr ( optimizers , args . optimizer ) optimize_op = Optimizer ( args . learning_rate ) . minimize ( model . valid_seqwise_loss_for_opt ) def _error_rate ( valid_predictions , valid_targets ) : incorrect_mask = tf . logical_not ( tf . equal ( tf . argmax ( valid_predictions , 1 ) , tf . argmax ( valid_targets , 1 ) ) ) return tf . reduce_mean ( tf . to_float ( incorrect_mask ) ) model . error_rate = _error_rate ( model . valid_predictions , model . valid_targets ) tf . summary . scalar ( 'train loss' , model . valid_seqwise_loss , collections = [ 'train' ] ) tf . summary . scalar ( 'train error rate' , model . error_rate , collections = [ 'train' ] ) model . val_error_rate = tf . placeholder ( tf . float32 , name = 'val_error_rate' ) tf . summary . scalar ( 'val error rate' , model . val_error_rate , collections = [ 'val' ] ) train_summary_op = tf . summary . merge_all ( 'train' ) val_summary_op = tf . summary . merge_all ( 'val' ) config = tf . ConfigProto ( ) config . gpu_options . allow_growth = True sess = tf . Session ( config = config ) sess . run ( tf . global_variables_initializer ( ) ) file_writer = tf . summary . FileWriter ( save_dir , graph = sess . graph , flush_secs = 10 ) saver = tf . train . Saver ( ) best_val_error_rate = 1.0 start_time = time . time ( ) for step in range ( NUM_OPT_STEPS ) : batch_inputs , batch_labels = next ( train_batches ) batch_targets = utils . one_hot ( np . squeeze ( batch_labels , 2 ) , TARGET_SIZE ) sess . run ( optimize_op , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0 : error_rate , summary = sess . run ( [ model . error_rate , train_summary_op ] , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) file_writer . add_summary ( summary , global_step = step ) with open ( os . path . join ( save_dir , 'train_status.txt' ) , 'a' ) as f : line = '%s %06.1f %d %.4f' % ( params_str , time . time ( ) - start_time , step , error_rate ) print ( line , file = f ) if step % NUM_STEPS_PER_VAL_SUMMARY == 0 : val_batches = utils . full_bptt_batch_generator ( val_inputs , val_labels , VAL_BATCH_SIZE , num_epochs = 1 , shuffle = False ) error_rates = [ ] for i , ( batch_inputs , batch_labels ) in enumerate ( val_batches ) : batch_targets = utils . one_hot ( np . squeeze ( batch_labels , 2 ) , TARGET_SIZE ) valid_predictions , valid_targets , batch_error_rates = sess . run ( [ model . valid_predictions , model . valid_targets , model . error_rate ] , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) error_rates . append ( batch_error_rates ) if args . debug and i == 0 : num_samples = 25 print ( 'Step: %d. Some targets and predictions:' % step ) print ( np . argmax ( valid_targets [ : num_samples ] , axis = 1 ) ) print ( np . argmax ( valid_predictions [ : num_samples ] , axis = 1 ) ) val_error_rate = np . mean ( error_rates , dtype = np . float ) if val_error_rate < best_val_error_rate : best_val_error_rate = val_error_rate saver . save ( sess , os . path . join ( save_dir , 'model.ckpt' ) ) summary = sess . run ( val_summary_op , feed_dict = { model . val_error_rate : val_error_rate } ) file_writer . add_summary ( summary , global_step = step ) with open ( os . path . join ( save_dir , 'val_status.txt' ) , 'a' ) as f : line = '%s %06.1f %d %.4f %.4f' % ( params_str , time . time ( ) - start_time , step , val_error_rate , best_val_error_rate ) print ( line , file = f ) if args . debug : print ( line ) file_writer . close ( )
Train an RNN for sequential (possibly permuted) MNIST recognition.
Train an RNN for sequential ( possibly permuted ) MNIST recognition .
""" Train an RNN for sequential (possibly permuted) MNIST recognition. """ # Flatten the images. # Align sequence-level labels with the appropriate time steps by padding with NaNs, # and to do so, first convert the labels to floats. # This is approximate if the validation batch size doesn't evenly divide # the number of validation examples.