class_docstr: TFMA benchmark base class.
class_docstr_tok: ["TFMA","benchmark","base","class","."]
docstr: Benchmark using the EvalSavedModel to compute metrics.
  
  Runs EvalSavedModel.metrics_reset_update_get_list and records the wall time
  taken.
docstr_tok: ["Benchmark","using","the","EvalSavedModel","to","compute","metrics",".","Runs","EvalSavedModel.metrics_reset_update_get_list","and","records","the","wall","time","taken","."]
code: def benchmarkEvalSavedModelMetricsResetUpdateGetList(self):
      """Benchmark using the EvalSavedModel to compute metrics.
  
      Runs EvalSavedModel.metrics_reset_update_get_list and records the wall time
      taken.
      """
      batch_size = 1000
  
      eval_saved_model = load.EvalSavedModel(
          path=self._dataset.tfma_saved_model_path(),
          include_default_metrics=True,
      )
  
      records = self._dataset.read_raw_dataset(
          deserialize=False, limit=self._max_num_examples()
      )
  
      start = time.time()
      accumulators = []
      for batch in benchmark_utils.batched_iterator(
          records, batch_size
      ):
          accumulators.append(
              eval_saved_model.metrics_reset_update_get_list(batch)
          )
      end = time.time()
      delta = end - start
  
      # Sanity check
      metric_variables_sum = accumulators[0]
      for acc in accumulators[1:]:
          if len(metric_variables_sum) != len(acc):
              raise ValueError(
                  "all metric variable value lists should have the same length, but "
                  "got lists with different lengths: %d and %d"
                  % (len(metric_variables_sum), len(acc))
              )
          metric_variables_sum = [
              a + b for a, b in zip(metric_variables_sum, acc)
          ]
  
      metrics = eval_saved_model.metrics_set_variables_and_get_values(
          metric_variables_sum
      )
      if "average_loss" not in metrics:
          raise ValueError(
              "metrics should contain average_loss metric, but it did not. "
              "metrics were: %s" % metrics
          )
  
      self.report_benchmark(
          iters=1,
          wall_time=delta,
          extras={
              "batch_size": batch_size,
              "num_examples": self._dataset.num_examples(
                  limit=self._max_num_examples()
              ),
          },
      )
  
file_path: tfx/tfx/benchmarks/tfma_benchmark_base.py
imports: import time
  import apache_beam
  import tensorflow_model_analysis
  from tensorflow_model_analysis.eval_saved_model import load
  import tfx
  from tfx.benchmarks import benchmark_utils
  from tfx.benchmarks import benchmark_base
label_desc: Use image node_id 8 for calling the TFMABenchmarkBase obj's underlying member method code with example usage: obj.benchmarkEvalSavedModelMetricsResetUpdateGetList() without return types