text: # coding=utf-8
  # Copyright 2018 The Google AI Language Team Authors.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """BERT finetuning runner."""
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import mmpy
  
  from run_classifier_common import *
  
  flags = tf.flags
  
  FLAGS = flags.FLAGS
  
  ## Required parameters
  flags.DEFINE_string(
      "data_dir", None,
      "The input data dir. Should contain the .tsv files (or other data files) "
      "for the task.")
  
  flags.DEFINE_string(
      "records_dir", None,
      "TF Records will be written and read from this directory. ")
  
  flags.DEFINE_string(
      "bert_config_file", None,
      "The config json file corresponding to the pre-trained BERT model. "
      "This specifies the model architecture.")
  
  flags.DEFINE_string("task_name", None, "The name of the task to train.")
  
  flags.DEFINE_string("vocab_file", None,
                      "The vocabulary file that the BERT model was trained on.")
  flags.DEFINE_string("placeholders_file", None,
                      "The vocabulary file containing placeholder tokens which "
                      "appear in the training text.")
  
  flags.DEFINE_string(
      "output_dir", None,
      "The output directory where the model checkpoints will be written.")
  
  ## Other parameters
  
  flags.DEFINE_string(
      "init_checkpoint", None,
      "Initial checkpoint (usually from a pre-trained BERT model).")
  
  flags.DEFINE_bool(
      "do_lower_case", True,
      "Whether to lower case the input text. Should be True for uncased "
      "models and False for cased models.")
  
  flags.DEFINE_integer(
      "max_seq_length", 128,
      "The maximum total input sequence length after WordPiece tokenization. "
      "Sequences longer than this will be truncated, and sequences shorter "
      "than this will be padded.")
  
  flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")
  
  flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.")
  
  flags.DEFINE_float("num_train_epochs", 3.0,
                     "Total number of training epochs to perform.")
  
  flags.DEFINE_float(
      "neg_sample_rate", 1.0,
      "Rate at which to include negative examples. "
      "1.0 -> always include")
  
  flags.DEFINE_float(
      "warmup_proportion", 0.1,
      "Proportion of training to perform linear learning rate warmup for. "
      "E.g., 0.1 = 10% of training.")
  
  flags.DEFINE_integer("save_checkpoints_steps", 1000,
                       "How often to save the model checkpoint.")
  
  flags.DEFINE_integer("iterations_per_loop", 1000,
                       "How many steps to make in each estimator call.")
  
  flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")
  
  tf.flags.DEFINE_string(
      "tpu_name", None,
      "The Cloud TPU to use for training. This should be either the name "
      "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
      "url.")
  
  tf.flags.DEFINE_string(
      "tpu_zone", None,
      "[Optional] GCE zone where the Cloud TPU is located in. If not "
      "specified, we will attempt to automatically detect the GCE project from "
      "metadata.")
  
  tf.flags.DEFINE_string(
      "gcp_project", None,
      "[Optional] Project name for the Cloud TPU-enabled project. If not "
      "specified, we will attempt to automatically detect the GCE project from "
      "metadata.")
  
  tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.")
  
  flags.DEFINE_integer(
      "num_tpu_cores", 8,
      "Only used if `use_tpu` is True. Total number of TPU cores to use.")
  
  
  def main(_):
    tf.logging.set_verbosity(tf.logging.INFO)
  
    tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,
                                                  FLAGS.init_checkpoint)
  
    bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
  
    if FLAGS.max_seq_length > bert_config.max_position_embeddings:
      raise ValueError(
          "Cannot use sequence length %d because the BERT model "
          "was only trained up to sequence length %d" %
          (FLAGS.max_seq_length, bert_config.max_position_embeddings))
  
    tf.gfile.MakeDirs(FLAGS.output_dir)
  
    task_name = FLAGS.task_name.lower()
  
    if task_name not in PROCESSORS:
      raise ValueError("Task not found: %s" % (task_name))
  
    processor = PROCESSORS[task_name]()
  
    neg_sample_rate = FLAGS.neg_sample_rate
  
    assert 0 <= neg_sample_rate <= 1, \
      "Invalid value for neg_sample_rate: {}".format(neg_sample_rate)
  
    if neg_sample_rate < 1 and processor.get_negative_label() is None:
      raise AssertionError("Negative sampling only works for projects "
                           "with negative labels")
  
    label_list = processor.get_labels(FLAGS.data_dir)
    neg_label_idx = None
    if processor.get_negative_label() is not None:
      neg_label_idx = label_list.index(processor.get_negative_label())
  
    tokenizer = tokenization.FullTokenizer(
        vocab_file=FLAGS.vocab_file,
        do_lower_case=FLAGS.do_lower_case,
        placeholders_file=FLAGS.placeholders_file)
  
    tpu_cluster_resolver = None
    if FLAGS.use_tpu and FLAGS.tpu_name:
      tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
          FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
  
    # use this by default, but over-ride if we are training. Save once per epoch
    save_checkpoint_steps = FLAGS.save_checkpoints_steps
  
    train_examples = None
    num_train_steps = None
    num_warmup_steps = None
  
    train_examples = processor.get_train_examples(FLAGS.data_dir)
    steps_per_epoch = len(train_examples) // FLAGS.train_batch_size
    num_train_steps = steps_per_epoch * FLAGS.num_train_epochs
    save_checkpoint_steps = steps_per_epoch
    tf.logging.info("Saving checkpoint every %d/%d steps",
                    steps_per_epoch, num_train_steps)
    num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
  
    is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
    run_config = tf.contrib.tpu.RunConfig(
        cluster=tpu_cluster_resolver,
        master=FLAGS.master,
        model_dir=FLAGS.output_dir,
        save_checkpoints_steps=save_checkpoint_steps,
        keep_checkpoint_max=100,
        tpu_config=tf.contrib.tpu.TPUConfig(
            iterations_per_loop=FLAGS.iterations_per_loop,
            num_shards=FLAGS.num_tpu_cores,
            per_host_input_for_training=is_per_host))
  
    # Default
    init_checkpoint = FLAGS.init_checkpoint
  
    model_fn = model_fn_builder(
        bert_config=bert_config,
        num_labels=len(label_list),
        negative_label_idx=neg_label_idx,
        init_checkpoint=init_checkpoint,
        learning_rate=FLAGS.learning_rate,
        num_train_steps=num_train_steps,
        num_warmup_steps=num_warmup_steps,
        use_tpu=FLAGS.use_tpu,
        use_one_hot_embeddings=FLAGS.use_tpu)
  
    # If TPU is not available, this will fall back to normal Estimator on CPU
    # or GPU.
    estimator = tf.contrib.tpu.TPUEstimator(
        use_tpu=FLAGS.use_tpu,
        model_fn=model_fn,
        config=run_config,
        train_batch_size=FLAGS.train_batch_size)
  
    records_dir = FLAGS.records_dir
    if records_dir is None:
      records_dir = FLAGS.data_dir
  
    train_file = os.path.join(records_dir, "train.tf_record")
    if os.path.exists(train_file):
      tf.logging.info("train file (%s) already exists -- no overwriting.",
                      train_file)
    else:
      file_based_convert_examples_to_features(
          train_examples, label_list, FLAGS.max_seq_length, tokenizer,
          train_file)
    tf.logging.info("***** Running training *****")
    tf.logging.info("  Num examples = %d", len(train_examples))
    tf.logging.info("  Batch size = %d", FLAGS.train_batch_size)
    tf.logging.info("  Num steps = %d", num_train_steps)
  
    if neg_sample_rate < 1:
      tf.logging.info("  Neg sample rate = %.2f", neg_sample_rate)
      train_input_fn = sampled_file_based_input_fn_builder(
          input_file=train_file,
          seq_length=FLAGS.max_seq_length,
          drop_remainder=True,
          neg_sample_rate=neg_sample_rate,
          neg_label_index=neg_label_idx
      )
    else:
      train_input_fn = file_based_input_fn_builder(
          input_file=train_file,
          seq_length=FLAGS.max_seq_length,
          is_training=True,
          drop_remainder=True)
  
    mmpy.mask_available_gpu(raise_if_none=True)
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
  
  
  if __name__ == "__main__":
    flags.mark_flag_as_required("data_dir")
    flags.mark_flag_as_required("task_name")
    flags.mark_flag_as_required("vocab_file")
    flags.mark_flag_as_required("bert_config_file")
    flags.mark_flag_as_required("output_dir")
    tf.app.run()
  