import tensorflow as tf


  if FLAGS.use_hvd:
    hvd.init()

    if FLAGS.reduce_log and (hvd.rank() != 0):
      tf.logging.set_verbosity(tf.logging.ERROR)

    FLAGS.output_dir = FLAGS.output_dir if hvd.rank() == 0 else os.path.join(FLAGS.output_dir, str(hvd.rank()))

  if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_train_eval:
    raise ValueError("At least one of `do_train` or `do_eval` must be True.")

  bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)

  tf.gfile.MakeDirs(FLAGS.output_dir)

  if FLAGS.recover_dir is not None:
    if FLAGS.use_hvd:
      FLAGS.recover_dir = FLAGS.recover_dir if hvd.rank() == 0 else os.path.join(FLAGS.recover_dir, str(hvd.rank()))
    path_ckpt = os.path.join(FLAGS.output_dir, "checkpoint")
    path_ckpt_input = os.path.join(FLAGS.output_dir, "checkpoint_input")

    if FLAGS.ckpt_no is not None and not tf.gfile.Exists(path_ckpt):
      with tf.gfile.GFile(path_ckpt, "w") as writer:
        writer.write('model_checkpoint_path: "%s-%s"\n' % (os.path.join(FLAGS.recover_dir, "model.ckpt"), str(FLAGS.ckpt_no)))
        writer.write('all_model_checkpoint_paths: "%s-%s"\n' % (os.path.join(FLAGS.recover_dir, "model.ckpt"), str(FLAGS.ckpt_no)))

    if FLAGS.ckpt_no_input is not None and not tf.gfile.Exists(path_ckpt_input):
