deepvariant/model_eval.py
def eval_loop(master,
              dataset_config_pbtxt,
              checkpoint_dir,
              model_name,
              batch_size,
              max_examples,
              eval_name,
              max_evaluations,
              use_tpu=False):
  """Evaluate incoming checkpoints, until the specified end."""
  logging.info('Running fixed eval for: %s', dataset_config_pbtxt)

  tf_dataset = data_providers.get_input_fn_from_dataset(
      dataset_config_filename=dataset_config_pbtxt,
      mode=tf.estimator.ModeKeys.EVAL,
      use_tpu=use_tpu,
  )

  best_ckpt = None
  ckpt_metric = FLAGS.best_checkpoint_metric
  ckpt_metric_increasing = ckpt_metric in increasing_metrics

  model = modeling.get_model(model_name)
  logging.info('Running evaluations on %s with model %s', tf_dataset, model)

  # Compute when to stop reading, in terms of batches.
  num_examples = tf_dataset.num_examples
  if max_examples is not None:
    num_examples = min(max_examples, num_examples)
  num_batches = num_examples // batch_size
  num_samples = batch_size * num_batches
  logging.info(
      'Dataset has %s samples, doing eval over %s; '
      'max_examples is %s, num examples to be used %s; num_batches is %s',
      tf_dataset.num_examples, num_samples, max_examples, num_examples,
      num_batches)

  # This loads EMA variables.
  eval_hooks = [h(checkpoint_dir) for h in model.session_eval_hooks()]

  classifier = model.make_estimator(
      batch_size=batch_size,
      model_dir=checkpoint_dir,
      use_tpu=use_tpu,
      master=master)

  def terminate_eval():
    logging.info('Terminating eval after %d seconds of no checkpoints',
                 FLAGS.eval_timeout)
    return True

  # Run evaluation when there's a new checkpoint
  num_evaluations = 0
  for ckpt in checkpoints_iterator(
      checkpoint_dir=checkpoint_dir,
      min_interval_secs=FLAGS.min_eval_interval_s,
      timeout=FLAGS.eval_timeout,
      timeout_fn=terminate_eval):

    logging.info('Starting to evaluate.')

    # For each step, calls input_fn, which returns one batch of data.
    # Evaluates until either steps batches are processed, or input_fn raises an
    # end-of-input exception (OutOfRangeError or StopIteration).
    eval_results = classifier.evaluate(
        input_fn=tf_dataset,
        steps=num_batches,
        hooks=eval_hooks,
        checkpoint_path=ckpt,
        name=eval_name)
    logging.info('Eval results: %s', eval_results)

    # Track best checkpoint seen so far, measured by ckpt_metric.
    if not best_ckpt:
      # If the training jobs died, pick up where we left off.
      try:
        best_metrics = read_metrics(ckpt, eval_name, 'best_checkpoint.metrics')
        logging.info('Found existing best_checkpoint: %s', best_metrics)
        best_ckpt = (best_metrics, ckpt)
      except NotFoundError:
        logging.info('best_checkpoint file does not exist.')
        best_ckpt = (eval_results, ckpt)
        _write_best_checkpoint(ckpt, eval_results, eval_name)
    if ((ckpt_metric_increasing and
         eval_results[ckpt_metric] > best_ckpt[0][ckpt_metric]) or
        (not ckpt_metric_increasing and
         eval_results[ckpt_metric] < best_ckpt[0][ckpt_metric])):
      best_ckpt = (eval_results, ckpt)
      _write_best_checkpoint(ckpt, eval_results, eval_name)

    _write_checkpoint_metrics(ckpt, eval_results, eval_name)

    # An alternative strategy might check step-number-of-ckpt >= train_steps.
    num_evaluations += 1
    if max_evaluations is not None and num_evaluations >= max_evaluations:
      logging.info('Evaluation finished after %d evaluations', num_evaluations)
      break

  return
Evaluate incoming checkpoints, until the specified end.
Evaluate incoming checkpoints, until the specified end.
Evaluate incoming checkpoints until the specified end .
def eval_loop(master,
              dataset_config_pbtxt,
              checkpoint_dir,
              model_name,
              batch_size,
              max_examples,
              eval_name,
              max_evaluations,
              use_tpu=False):
  logging.info('Running fixed eval for: %s', dataset_config_pbtxt)
  tf_dataset = data_providers.get_input_fn_from_dataset(
      dataset_config_filename=dataset_config_pbtxt,
      mode=tf.estimator.ModeKeys.EVAL,
      use_tpu=use_tpu,
  )
  best_ckpt = None
  ckpt_metric = FLAGS.best_checkpoint_metric
  ckpt_metric_increasing = ckpt_metric in increasing_metrics
  model = modeling.get_model(model_name)
  logging.info('Running evaluations on %s with model %s', tf_dataset, model)
  num_examples = tf_dataset.num_examples
  if max_examples is not None:
    num_examples = min(max_examples, num_examples)
  num_batches = num_examples // batch_size
  num_samples = batch_size * num_batches
  logging.info(
      'Dataset has %s samples, doing eval over %s; '
      'max_examples is %s, num examples to be used %s; num_batches is %s',
      tf_dataset.num_examples, num_samples, max_examples, num_examples,
      num_batches)
  eval_hooks = [h(checkpoint_dir) for h in model.session_eval_hooks()]
  classifier = model.make_estimator(
      batch_size=batch_size,
      model_dir=checkpoint_dir,
      use_tpu=use_tpu,
      master=master)
  def terminate_eval():
    logging.info('Terminating eval after %d seconds of no checkpoints',
                 FLAGS.eval_timeout)
    return True
  num_evaluations = 0
  for ckpt in checkpoints_iterator(
      checkpoint_dir=checkpoint_dir,
      min_interval_secs=FLAGS.min_eval_interval_s,
      timeout=FLAGS.eval_timeout,
      timeout_fn=terminate_eval):
    logging.info('Starting to evaluate.')
    eval_results = classifier.evaluate(
        input_fn=tf_dataset,
        steps=num_batches,
        hooks=eval_hooks,
        checkpoint_path=ckpt,
        name=eval_name)
    logging.info('Eval results: %s', eval_results)
    if not best_ckpt:
      try:
        best_metrics = read_metrics(ckpt, eval_name, 'best_checkpoint.metrics')
        logging.info('Found existing best_checkpoint: %s', best_metrics)
        best_ckpt = (best_metrics, ckpt)
      except NotFoundError:
        logging.info('best_checkpoint file does not exist.')
        best_ckpt = (eval_results, ckpt)
        _write_best_checkpoint(ckpt, eval_results, eval_name)
    if ((ckpt_metric_increasing and
         eval_results[ckpt_metric] > best_ckpt[0][ckpt_metric]) or
        (not ckpt_metric_increasing and
         eval_results[ckpt_metric] < best_ckpt[0][ckpt_metric])):
      best_ckpt = (eval_results, ckpt)
      _write_best_checkpoint(ckpt, eval_results, eval_name)
    _write_checkpoint_metrics(ckpt, eval_results, eval_name)
    num_evaluations += 1
    if max_evaluations is not None and num_evaluations >= max_evaluations:
      logging.info('Evaluation finished after %d evaluations', num_evaluations)
      break
  return
def eval_loop ( master , dataset_config_pbtxt , checkpoint_dir , model_name , batch_size , max_examples , eval_name , max_evaluations , use_tpu = False ) : logging . info ( 'Running fixed eval for: %s' , dataset_config_pbtxt ) tf_dataset = data_providers . get_input_fn_from_dataset ( dataset_config_filename = dataset_config_pbtxt , mode = tf . estimator . ModeKeys . EVAL , use_tpu = use_tpu , ) best_ckpt = None ckpt_metric = FLAGS . best_checkpoint_metric ckpt_metric_increasing = ckpt_metric in increasing_metrics model = modeling . get_model ( model_name ) logging . info ( 'Running evaluations on %s with model %s' , tf_dataset , model ) num_examples = tf_dataset . num_examples if max_examples is not None : num_examples = min ( max_examples , num_examples ) num_batches = num_examples // batch_size num_samples = batch_size * num_batches logging . info ( 'Dataset has %s samples, doing eval over %s; ' 'max_examples is %s, num examples to be used %s; num_batches is %s' , tf_dataset . num_examples , num_samples , max_examples , num_examples , num_batches ) eval_hooks = [ h ( checkpoint_dir ) for h in model . session_eval_hooks ( ) ] classifier = model . make_estimator ( batch_size = batch_size , model_dir = checkpoint_dir , use_tpu = use_tpu , master = master ) def terminate_eval ( ) : logging . info ( 'Terminating eval after %d seconds of no checkpoints' , FLAGS . eval_timeout ) return True num_evaluations = 0 for ckpt in checkpoints_iterator ( checkpoint_dir = checkpoint_dir , min_interval_secs = FLAGS . min_eval_interval_s , timeout = FLAGS . eval_timeout , timeout_fn = terminate_eval ) : logging . info ( 'Starting to evaluate.' ) eval_results = classifier . evaluate ( input_fn = tf_dataset , steps = num_batches , hooks = eval_hooks , checkpoint_path = ckpt , name = eval_name ) logging . info ( 'Eval results: %s' , eval_results ) if not best_ckpt : try : best_metrics = read_metrics ( ckpt , eval_name , 'best_checkpoint.metrics' ) logging . info ( 'Found existing best_checkpoint: %s' , best_metrics ) best_ckpt = ( best_metrics , ckpt ) except NotFoundError : logging . info ( 'best_checkpoint file does not exist.' ) best_ckpt = ( eval_results , ckpt ) _write_best_checkpoint ( ckpt , eval_results , eval_name ) if ( ( ckpt_metric_increasing and eval_results [ ckpt_metric ] > best_ckpt [ 0 ] [ ckpt_metric ] ) or ( not ckpt_metric_increasing and eval_results [ ckpt_metric ] < best_ckpt [ 0 ] [ ckpt_metric ] ) ) : best_ckpt = ( eval_results , ckpt ) _write_best_checkpoint ( ckpt , eval_results , eval_name ) _write_checkpoint_metrics ( ckpt , eval_results , eval_name ) num_evaluations += 1 if max_evaluations is not None and num_evaluations >= max_evaluations : logging . info ( 'Evaluation finished after %d evaluations' , num_evaluations ) break return
Evaluate incoming checkpoints, until the specified end.
Evaluate incoming checkpoints until the specified end .
"""Evaluate incoming checkpoints, until the specified end.""" # Compute when to stop reading, in terms of batches. # This loads EMA variables. # Run evaluation when there's a new checkpoint # For each step, calls input_fn, which returns one batch of data. # Evaluates until either steps batches are processed, or input_fn raises an # end-of-input exception (OutOfRangeError or StopIteration). # Track best checkpoint seen so far, measured by ckpt_metric. # If the training jobs died, pick up where we left off. # An alternative strategy might check step-number-of-ckpt >= train_steps.
master
dataset_config_pbtxt
checkpoint_dir
model_name
batch_size
max_examples
eval_name
max_evaluations
use_tpu