code: def aggregate_stats(stats_files):
      """Aggregate stats in per-shard stats files."""
      all_stats = {}
      for fname in stats_files:
          with tf.gfile.Open(fname) as f:
              stats = json.loads(f.read())
              for k, v in six.iteritems(stats):
                  if k not in all_stats:
                      if isinstance(v, list):
                          all_stats[k] = []
                      else:
                          all_stats[k] = 0
  
                  if isinstance(v, list):
                      all_stats[k].extend(v)
                  else:
                      all_stats[k] += v
  
      stats = all_stats
      ref_coverage = (
          float(stats["total_found_refs"])
          / stats["total_original_refs"]
      )
      len_bounds = [
          0,
          2,
          10,
          100,
          1000,
          5000,
          10000,
          20000,
          50000,
          100000,
          1000000,
      ]
      len_counts, len_bounds = np.histogram(
          stats["ref_lengths"], len_bounds
      )
      len_dist = len_counts.astype(np.float32) / len_counts.sum()
      wiki_coverage = (
          float(stats["num_wikis_written"])
          / stats["total_original_wikis"]
      )
      wikis_skipped_no_ref = (
          float(stats["wikis_skipped_no_refs"])
          / stats["total_original_wikis"]
      )
      wikis_skipped_no_lead = (
          float(stats["wikis_skipped_short_lead"])
          / stats["total_original_wikis"]
      )
      wiki_ref_coverage = [
          float(found) / orig
          for found, orig in zip(
              stats["wiki_found_refs"], stats["wiki_original_refs"]
          )
          if found
      ]
      coverage_bounds = np.arange(21).astype(np.float32) / 20
      coverage_counts, coverage_bounds = np.histogram(
          wiki_ref_coverage, coverage_bounds
      )
      coverage_dist = (
          coverage_counts.astype(np.float32) / coverage_counts.sum()
      )
  
      agg_stats = dict(
          total_original_wikis=stats["total_original_wikis"],
          total_original_refs=stats["total_original_refs"],
          wiki_coverage=wiki_coverage,
          wikis_skipped_no_ref=wikis_skipped_no_ref,
          wikis_skipped_no_lead=wikis_skipped_no_lead,
          overall_ref_coverage=ref_coverage,
          per_wiki_ref_coverage_dist=list(
              (coverage_dist * 100).astype(int)
          ),
          per_wiki_ref_coverage_bounds=list(
              (coverage_bounds * 100).astype(int)
          ),
          ref_len_dist=list((len_dist * 100).astype(int)),
          ref_len_bounds=list(len_bounds),
      )
      return agg_stats
  
file_path: tensor2tensor/tensor2tensor/data_generators/wikisum/validate_data.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import json
  import os
  import numpy
  import six
  from six.moves import zip
  from tensor2tensor.data_generators.wikisum import wikisum
  import tensorflow.compat.v1
label_desc: Use image node_id 1 for calling a global function with example usage: aggregate_stats(stats_files) and returns: agg_stats