text: # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  
  r"""Evaluation executable for detection models.
  
  This executable is used to evaluate DetectionModels. There are two ways of
  configuring the eval job.
  
  1) A single pipeline_pb2.TrainEvalPipelineConfig file maybe specified instead.
  In this mode, the --eval_training_data flag may be given to force the pipeline
  to evaluate on training data instead.
  
  Example usage:
      ./eval \
          --logtostderr \
          --checkpoint_dir=path/to/checkpoint_dir \
          --eval_dir=path/to/eval_dir \
          --pipeline_config_path=pipeline_config.pbtxt
  
  2) Three configuration files may be provided: a model_pb2.DetectionModel
  configuration file to define what type of DetectionModel is being evaluated, an
  input_reader_pb2.InputReader file to specify what data the model is evaluating
  and an eval_pb2.EvalConfig file to configure evaluation parameters.
  
  Example usage:
      ./eval \
          --logtostderr \
          --checkpoint_dir=path/to/checkpoint_dir \
          --eval_dir=path/to/eval_dir \
          --eval_config_path=eval_config.pbtxt \
          --model_config_path=model_config.pbtxt \
          --input_config_path=eval_input_config.pbtxt
  """
  import functools
  import tensorflow as tf
  
  from google.protobuf import text_format
  from object_detection import evaluator
  from object_detection.builders import input_reader_builder
  from object_detection.builders import model_builder
  from object_detection.protos import eval_pb2
  from object_detection.protos import input_reader_pb2
  from object_detection.protos import model_pb2
  from object_detection.protos import pipeline_pb2
  from object_detection.utils import label_map_util
  
  tf.logging.set_verbosity(tf.logging.INFO)
  
  flags = tf.app.flags
  flags.DEFINE_boolean('eval_training_data', False,
                       'If training data should be evaluated for this job.')
  flags.DEFINE_string('checkpoint_dir', '',
                      'Directory containing checkpoints to evaluate, typically '
                      'set to `train_dir` used in the training job.')
  flags.DEFINE_string('eval_dir', '',
                      'Directory to write eval summaries to.')
  flags.DEFINE_string('pipeline_config_path', '',
                      'Path to a pipeline_pb2.TrainEvalPipelineConfig config '
                      'file. If provided, other configs are ignored')
  flags.DEFINE_string('eval_config_path', '',
                      'Path to an eval_pb2.EvalConfig config file.')
  flags.DEFINE_string('input_config_path', '',
                      'Path to an input_reader_pb2.InputReader config file.')
  flags.DEFINE_string('model_config_path', '',
                      'Path to a model_pb2.DetectionModel config file.')
  
  FLAGS = flags.FLAGS
  
  
  def get_configs_from_pipeline_file():
    """Reads evaluation configuration from a pipeline_pb2.TrainEvalPipelineConfig.
  
    Reads evaluation config from file specified by pipeline_config_path flag.
  
    Returns:
      model_config: a model_pb2.DetectionModel
      eval_config: a eval_pb2.EvalConfig
      input_config: a input_reader_pb2.InputReader
    """
    pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
    with tf.gfile.GFile(FLAGS.pipeline_config_path, 'r') as f:
      text_format.Merge(f.read(), pipeline_config)
  
    model_config = pipeline_config.model
    if FLAGS.eval_training_data:
      eval_config = pipeline_config.train_config
    else:
      eval_config = pipeline_config.eval_config
    input_config = pipeline_config.eval_input_reader
  
    return model_config, eval_config, input_config
  
  
  def get_configs_from_multiple_files():
    """Reads evaluation configuration from multiple config files.
  
    Reads the evaluation config from the following files:
      model_config: Read from --model_config_path
      eval_config: Read from --eval_config_path
      input_config: Read from --input_config_path
  
    Returns:
      model_config: a model_pb2.DetectionModel
      eval_config: a eval_pb2.EvalConfig
      input_config: a input_reader_pb2.InputReader
    """
    eval_config = eval_pb2.EvalConfig()
    with tf.gfile.GFile(FLAGS.eval_config_path, 'r') as f:
      text_format.Merge(f.read(), eval_config)
  
    model_config = model_pb2.DetectionModel()
    with tf.gfile.GFile(FLAGS.model_config_path, 'r') as f:
      text_format.Merge(f.read(), model_config)
  
    input_config = input_reader_pb2.InputReader()
    with tf.gfile.GFile(FLAGS.input_config_path, 'r') as f:
      text_format.Merge(f.read(), input_config)
  
    return model_config, eval_config, input_config
  
  
  def main(unused_argv):
    assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
    assert FLAGS.eval_dir, '`eval_dir` is missing.'
    if FLAGS.pipeline_config_path:
      model_config, eval_config, input_config = get_configs_from_pipeline_file()
    else:
      model_config, eval_config, input_config = get_configs_from_multiple_files()
  
    model_fn = functools.partial(
        model_builder.build,
        model_config=model_config,
        is_training=False)
  
    create_input_dict_fn = functools.partial(
        input_reader_builder.build,
        input_config)
  
    label_map = label_map_util.load_labelmap(input_config.label_map_path)
    max_num_classes = max([item.id for item in label_map.item])
    categories = label_map_util.convert_label_map_to_categories(
        label_map, max_num_classes)
  
    evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
                       FLAGS.checkpoint_dir, FLAGS.eval_dir)
  
  
  if __name__ == '__main__':
    tf.app.run()
  