code: def main(_):
      if FLAGS.for_commoncrawl:
          problem = wikisum.WikisumCommoncrawl()
      else:
          problem = wikisum.WikisumWeb()
      prefix = problem.dataset_filename()
      data_files = tf.gfile.Glob(
          os.path.join(FLAGS.out_dir, "%s*" % prefix)
      )
      missing_files = validate_data_files(
          problem,
          data_files,
          min_size=(60 if FLAGS.for_commoncrawl else 120) * 1e6,
      )
  
      task_ids = [filename_to_task_id(fname) for fname in missing_files]
      ids_for_flag = ",".join([str(i) for i in task_ids])
      tf.logging.error(
          "You should (re)generate %d of the data files. "
          "Rerun produce_examples with --instance_ids='%s'.",
          len(missing_files),
          ids_for_flag,
      )
  
      # Compute and write out aggregated stats
      stats_files = tf.gfile.Glob(os.path.join(FLAGS.out_dir, "stats*"))
      agg_stats = aggregate_stats(stats_files)
      if not FLAGS.for_commoncrawl:
          coverage = agg_stats["overall_ref_coverage"] * 100
          if not coverage > 80:
              tf.logging.error(
                  "Overall reference coverage is expected to be > 80%. "
                  "It is %0.1f. You may want to rerun get_references_web.",
                  coverage,
              )
      with tf.gfile.Open(
          os.path.join(FLAGS.out_dir, "stats.json"), "w"
      ) as f:
          f.write(json.dumps(agg_stats))
      if FLAGS.rm_per_shard_stats and not missing_files:
          for fname in stats_files:
              tf.gfile.Remove(fname)
  
file_path: tensor2tensor/tensor2tensor/data_generators/wikisum/validate_data.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import json
  import os
  import numpy
  import six
  from six.moves import zip
  from tensor2tensor.data_generators.wikisum import wikisum
  import tensorflow.compat.v1
label_desc: Use image node_id 5 for calling a global function with example usage: main(_) without return types