additionproblem_train.py
def main():
  """ Train an RNN for the Addition Problem. """

  args, params_str, layer_kwargs = parse_args()

  save_dir = os.path.join(args.data_dir, 'results', params_str)
  if os.path.exists(save_dir):
    shutil.rmtree(save_dir)
  os.makedirs(save_dir)

  # inputs are [num_examples, length, 2]; targets are [num_examples]. We need
  # everything to be 3-D for the batch generator, and we need to align the
  # targets with the last time step.
  outs = additionproblem.load_split(args.data_dir, args.length, val=True)
  train_inputs, train_targets, val_inputs, val_targets = outs
  pad_width = [[0, 0], [train_inputs.shape[1] - 1, 0], [0, 0]]
  train_targets = train_targets[:, np.newaxis, np.newaxis]
  train_targets = np.pad(train_targets, pad_width, mode='constant', constant_values=np.nan)
  val_targets = val_targets[:, np.newaxis, np.newaxis]
  val_targets = np.pad(val_targets, pad_width, mode='constant', constant_values=np.nan)

  train_batches = utils.full_bptt_batch_generator(train_inputs, train_targets, BATCH_SIZE, shuffle=True)

  model = models.RNNRegressionModel(args.layer_type, INPUT_SIZE, TARGET_SIZE, args.num_hidden_units,
                                    args.activation_type, **layer_kwargs)

  Optimizer = getattr(optimizers, args.optimizer)
  optimizer = Optimizer(args.learning_rate)
  optimize_op = optimizer.minimize(model.valid_stepwise_loss_for_opt)

  tf.summary.scalar('train mse', model.valid_stepwise_loss, collections=['train'])

  model.val_loss = tf.placeholder(tf.float32, shape=[], name='val_loss')
  tf.summary.scalar('val mse', model.val_loss, collections=['val'])

  train_summary_op = tf.summary.merge_all('train')
  val_summary_op = tf.summary.merge_all('val')

  config = tf.ConfigProto()
  config.gpu_options.allow_growth = False
  sess = tf.Session(config=config)
  sess.run(tf.global_variables_initializer())

  file_writer = tf.summary.FileWriter(save_dir, graph=sess.graph, flush_secs=10)
  saver = tf.train.Saver()

  best_val_loss = np.inf
  start_time = time.time()
  for step in range(NUM_OPT_STEPS):

    batch_inputs, batch_targets = next(train_batches)

    sess.run(optimize_op,
             feed_dict={model.inputs: batch_inputs,
                        model.targets: batch_targets})

    if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0:

      loss, summary = sess.run([model.valid_stepwise_loss, train_summary_op],
                               feed_dict={model.inputs: batch_inputs,
                                          model.targets: batch_targets})

      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'train_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f' % (params_str, time.time() - start_time, step, loss)
        print(line, file=f)

    if step % NUM_STEPS_PER_VAL_SUMMARY == 0:

      val_batches = utils.full_bptt_batch_generator(val_inputs, val_targets, VAL_BATCH_SIZE, num_epochs=1,
                                                    shuffle=False)
      losses = []
      for batch_inputs, batch_targets in val_batches:
        valid_predictions, valid_targets, loss = sess.run(
          [model.valid_predictions, model.valid_targets, model.valid_stepwise_loss],
          feed_dict={model.inputs: batch_inputs,
                     model.targets: batch_targets}
        )
        losses.append(loss)

      val_loss = np.mean(losses, dtype=np.float)
      if val_loss < best_val_loss:
        best_val_loss = val_loss
        saver.save(sess, os.path.join(save_dir, 'model.ckpt'))

      summary = sess.run(val_summary_op, feed_dict={model.val_loss: val_loss})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'val_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f %.4f' % (params_str, time.time() - start_time, step,
                                           val_loss, best_val_loss)
        print(line, file=f)
        if args.debug:
          print(line)

  file_writer.close()
 Train an RNN for the Addition Problem. 
Train an RNN for the Addition Problem.
Train an RNN for the Addition Problem .
def main():
  args, params_str, layer_kwargs = parse_args()
  save_dir = os.path.join(args.data_dir, 'results', params_str)
  if os.path.exists(save_dir):
    shutil.rmtree(save_dir)
  os.makedirs(save_dir)
  outs = additionproblem.load_split(args.data_dir, args.length, val=True)
  train_inputs, train_targets, val_inputs, val_targets = outs
  pad_width = [[0, 0], [train_inputs.shape[1] - 1, 0], [0, 0]]
  train_targets = train_targets[:, np.newaxis, np.newaxis]
  train_targets = np.pad(train_targets, pad_width, mode='constant', constant_values=np.nan)
  val_targets = val_targets[:, np.newaxis, np.newaxis]
  val_targets = np.pad(val_targets, pad_width, mode='constant', constant_values=np.nan)
  train_batches = utils.full_bptt_batch_generator(train_inputs, train_targets, BATCH_SIZE, shuffle=True)
  model = models.RNNRegressionModel(args.layer_type, INPUT_SIZE, TARGET_SIZE, args.num_hidden_units,
                                    args.activation_type, **layer_kwargs)
  Optimizer = getattr(optimizers, args.optimizer)
  optimizer = Optimizer(args.learning_rate)
  optimize_op = optimizer.minimize(model.valid_stepwise_loss_for_opt)
  tf.summary.scalar('train mse', model.valid_stepwise_loss, collections=['train'])
  model.val_loss = tf.placeholder(tf.float32, shape=[], name='val_loss')
  tf.summary.scalar('val mse', model.val_loss, collections=['val'])
  train_summary_op = tf.summary.merge_all('train')
  val_summary_op = tf.summary.merge_all('val')
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = False
  sess = tf.Session(config=config)
  sess.run(tf.global_variables_initializer())
  file_writer = tf.summary.FileWriter(save_dir, graph=sess.graph, flush_secs=10)
  saver = tf.train.Saver()
  best_val_loss = np.inf
  start_time = time.time()
  for step in range(NUM_OPT_STEPS):
    batch_inputs, batch_targets = next(train_batches)
    sess.run(optimize_op,
             feed_dict={model.inputs: batch_inputs,
                        model.targets: batch_targets})
    if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0:
      loss, summary = sess.run([model.valid_stepwise_loss, train_summary_op],
                               feed_dict={model.inputs: batch_inputs,
                                          model.targets: batch_targets})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'train_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f' % (params_str, time.time() - start_time, step, loss)
        print(line, file=f)
    if step % NUM_STEPS_PER_VAL_SUMMARY == 0:
      val_batches = utils.full_bptt_batch_generator(val_inputs, val_targets, VAL_BATCH_SIZE, num_epochs=1,
                                                    shuffle=False)
      losses = []
      for batch_inputs, batch_targets in val_batches:
        valid_predictions, valid_targets, loss = sess.run(
          [model.valid_predictions, model.valid_targets, model.valid_stepwise_loss],
          feed_dict={model.inputs: batch_inputs,
                     model.targets: batch_targets}
        )
        losses.append(loss)
      val_loss = np.mean(losses, dtype=np.float)
      if val_loss < best_val_loss:
        best_val_loss = val_loss
        saver.save(sess, os.path.join(save_dir, 'model.ckpt'))
      summary = sess.run(val_summary_op, feed_dict={model.val_loss: val_loss})
      file_writer.add_summary(summary, global_step=step)
      with open(os.path.join(save_dir, 'val_status.txt'), 'a') as f:
        line = '%s %06.1f %d %.4f %.4f' % (params_str, time.time() - start_time, step,
                                           val_loss, best_val_loss)
        print(line, file=f)
        if args.debug:
          print(line)
  file_writer.close()
def main ( ) : args , params_str , layer_kwargs = parse_args ( ) save_dir = os . path . join ( args . data_dir , 'results' , params_str ) if os . path . exists ( save_dir ) : shutil . rmtree ( save_dir ) os . makedirs ( save_dir ) outs = additionproblem . load_split ( args . data_dir , args . length , val = True ) train_inputs , train_targets , val_inputs , val_targets = outs pad_width = [ [ 0 , 0 ] , [ train_inputs . shape [ 1 ] - 1 , 0 ] , [ 0 , 0 ] ] train_targets = train_targets [ : , np . newaxis , np . newaxis ] train_targets = np . pad ( train_targets , pad_width , mode = 'constant' , constant_values = np . nan ) val_targets = val_targets [ : , np . newaxis , np . newaxis ] val_targets = np . pad ( val_targets , pad_width , mode = 'constant' , constant_values = np . nan ) train_batches = utils . full_bptt_batch_generator ( train_inputs , train_targets , BATCH_SIZE , shuffle = True ) model = models . RNNRegressionModel ( args . layer_type , INPUT_SIZE , TARGET_SIZE , args . num_hidden_units , args . activation_type , ** layer_kwargs ) Optimizer = getattr ( optimizers , args . optimizer ) optimizer = Optimizer ( args . learning_rate ) optimize_op = optimizer . minimize ( model . valid_stepwise_loss_for_opt ) tf . summary . scalar ( 'train mse' , model . valid_stepwise_loss , collections = [ 'train' ] ) model . val_loss = tf . placeholder ( tf . float32 , shape = [ ] , name = 'val_loss' ) tf . summary . scalar ( 'val mse' , model . val_loss , collections = [ 'val' ] ) train_summary_op = tf . summary . merge_all ( 'train' ) val_summary_op = tf . summary . merge_all ( 'val' ) config = tf . ConfigProto ( ) config . gpu_options . allow_growth = False sess = tf . Session ( config = config ) sess . run ( tf . global_variables_initializer ( ) ) file_writer = tf . summary . FileWriter ( save_dir , graph = sess . graph , flush_secs = 10 ) saver = tf . train . Saver ( ) best_val_loss = np . inf start_time = time . time ( ) for step in range ( NUM_OPT_STEPS ) : batch_inputs , batch_targets = next ( train_batches ) sess . run ( optimize_op , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) if step % NUM_STEPS_PER_TRAIN_SUMMARY == 0 : loss , summary = sess . run ( [ model . valid_stepwise_loss , train_summary_op ] , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) file_writer . add_summary ( summary , global_step = step ) with open ( os . path . join ( save_dir , 'train_status.txt' ) , 'a' ) as f : line = '%s %06.1f %d %.4f' % ( params_str , time . time ( ) - start_time , step , loss ) print ( line , file = f ) if step % NUM_STEPS_PER_VAL_SUMMARY == 0 : val_batches = utils . full_bptt_batch_generator ( val_inputs , val_targets , VAL_BATCH_SIZE , num_epochs = 1 , shuffle = False ) losses = [ ] for batch_inputs , batch_targets in val_batches : valid_predictions , valid_targets , loss = sess . run ( [ model . valid_predictions , model . valid_targets , model . valid_stepwise_loss ] , feed_dict = { model . inputs : batch_inputs , model . targets : batch_targets } ) losses . append ( loss ) val_loss = np . mean ( losses , dtype = np . float ) if val_loss < best_val_loss : best_val_loss = val_loss saver . save ( sess , os . path . join ( save_dir , 'model.ckpt' ) ) summary = sess . run ( val_summary_op , feed_dict = { model . val_loss : val_loss } ) file_writer . add_summary ( summary , global_step = step ) with open ( os . path . join ( save_dir , 'val_status.txt' ) , 'a' ) as f : line = '%s %06.1f %d %.4f %.4f' % ( params_str , time . time ( ) - start_time , step , val_loss , best_val_loss ) print ( line , file = f ) if args . debug : print ( line ) file_writer . close ( )
Train an RNN for the Addition Problem.
Train an RNN for the Addition Problem .
""" Train an RNN for the Addition Problem. """ # inputs are [num_examples, length, 2]; targets are [num_examples]. We need # everything to be 3-D for the batch generator, and we need to align the # targets with the last time step.