import tensorflow as tf

            tf.summary.scalar('Learning rate', m.lr)

    latest_ckpt = tf.train.latest_checkpoint(FLAGS.save_path)

    with train_graph.as_default():
        sv = tf.train.Supervisor(logdir=FLAGS.save_path)
        config_proto = tf.ConfigProto(log_device_placement=False,
                                      allow_soft_placement=True)
        with sv.managed_session(config=config_proto) as train_sess:
        #with tf.Session(config=config_proto) as train_sess:
            train_sess.run(tf.global_variables_initializer())
            for i in range(config.max_max_epoch):
                lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.)
                m.assign_lr(train_sess, config.learning_rate * lr_decay)
                train_perplexity = run_epoch(train_sess, m, #eval_op=m.train_op,
                                             verbose=True)
                print('Epoch {} Train Perplexity: {:.3f}'.format(i + 1,
                                                                 train_perplexity))
                if i % 5 == 0:
                    sv.saver.save(train_sess, FLAGS.save_path,
