code: def main(_):
      assert FLAGS.out_dir
      assert FLAGS.metadata_dir
      out_dir = os.path.join(
          FLAGS.out_dir, "process_%d" % FLAGS.task_id
      )
      tf.gfile.MakeDirs(out_dir)
  
      with utils.timing("get_refs_commoncrawl"):
          # Get all WET files
          if FLAGS.commoncrawl_wet_dir:
              wet_files = tf.gfile.Glob(
                  os.path.join(FLAGS.commoncrawl_wet_dir, "*.wet.gz")
              )
          else:
              tmp_dir = tempfile.gettempdir()
              wet_files = list(
                  utils.wet_download_urls(
                      utils.WET_PATHS_BY_DATE["0917"], tmp_dir
                  )
              )
  
          # Shard and select this task's work
          wet_files.sort()
          wet_files = utils.shard(wet_files, FLAGS.num_tasks)[
              FLAGS.task_id
          ]
          tf.logging.info(
              "Sharded out WET files. Processing %d files",
              len(wet_files),
          )
  
          wikisum.extract_references_from_wets(
              wet_files, FLAGS.metadata_dir, out_dir
          )
  
file_path: tensor2tensor/tensor2tensor/data_generators/wikisum/get_references_commoncrawl.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import os
  import tempfile
  from tensor2tensor.data_generators.wikisum import utils
  from tensor2tensor.data_generators.wikisum import wikisum
  import tensorflow.compat.v1
label_desc: Use image node_id 1 for calling a global function with example usage: main(_) without return types