class_docstr: Old segment -> revised segment.
  
  Data filtration heuristics:
    wiki_revision_max_page_size_exp:
      pages above this # of bytes are thrown out
  
    wiki_revision_revision_skip_factor:
      rate of logarithmic downsampling of revision history list
  
    wiki_revision_percent_identical_examples:
      how many identitcal examples to admit, as percent of total examples
  
    wiki_revision_introduce_errors:
      whether or not to introduce spelling-type errors on the source side
  
    wiki_revision_max_equal_to_diff_ratio:
      whether or not to introduce spelling-type errors on the source side
  
  
  Vocab size=32k
  Maximum input/target length = 1024 wordpiece tokens
class_docstr_tok: ["Old","segment","-",">","revised","segment",".","Data","filtration","heuristics",":","wiki_revision_max_page_size_exp",":","pages","above","this","#","of","bytes","are","thrown","out","wiki_revision_revision_skip_factor",":","rate","of","logarithmic","downsampling","of","revision","history","list","wiki_revision_percent_identical_examples",":","how","many","identitcal","examples","to","admit",",","as","percent","of","total","examples","wiki_revision_introduce_errors",":","whether","or","not","to","introduce","spelling-type","errors","on","the","source","side","wiki_revision_max_equal_to_diff_ratio",":","whether","or","not","to","introduce","spelling-type","errors","on","the","source","side","Vocab","size=32k","Maximum","input\/target","length","=","1024","wordpiece","tokens"]
code: def generate_data(self, data_dir, tmp_dir, task_id=-1):
      if task_id == -1 or task_id is None:
          for i in range(
              FLAGS.wiki_revision_num_train_shards
              + FLAGS.wiki_revision_num_dev_shards
          ):
              self.generate_data(data_dir, tmp_dir, i)
              return
  
      tf.logging.info(
          "Flags for job (task_id {}): "
          "Dev shards: {}, Train shards: {}, "
          "Revision skip factor: {}, Max page size: 2**{}, Introduce errors: {},"
          "Percent Identical Examples: {}"
          "".format(
              task_id,
              FLAGS.wiki_revision_num_dev_shards,
              FLAGS.wiki_revision_num_train_shards,
              FLAGS.wiki_revision_revision_skip_factor,
              FLAGS.wiki_revision_max_page_size_exp,
              FLAGS.wiki_revision_introduce_errors,
              FLAGS.wiki_revision_percent_identical_examples,
          )
      )
  
      if FLAGS.wiki_revision_vocab_file:
          encoder = wiki_revision_utils.get_encoder_from_vocab(
              FLAGS.wiki_revision_vocab_file
          )
      else:
          encoder = wiki_revision_utils.get_or_generate_vocabulary(
              data_dir,
              tmp_dir,
              FLAGS.wiki_revision_data_prefix,
              FLAGS.wiki_revision_max_page_size_exp,
              self.approx_vocab_size,
              self.strip,
          )
  
      random.seed(123)
      if task_id < FLAGS.wiki_revision_num_train_shards:
          out_file = self.training_filepaths(
              data_dir,
              FLAGS.wiki_revision_num_train_shards,
              shuffled=False,
          )[task_id]
      else:
          out_file = self.dev_filepaths(
              data_dir,
              FLAGS.wiki_revision_num_dev_shards,
              shuffled=False,
          )[task_id - FLAGS.wiki_revision_num_train_shards]
  
      tf.logging.info("Generating files for path: %s", out_file)
      self.corpus_files = wiki_revision_utils.corpus_files_for_shard(
          task_id,
          FLAGS.wiki_revision_num_train_shards,
          FLAGS.wiki_revision_num_dev_shards,
          FLAGS.wiki_revision_data_prefix,
      )
      example_generator = self.generator(
          encoder, self.corpus_files, tmp_dir
      )
  
      packed_example_generator = self._maybe_pack_examples(
          example_generator
      )
      generator_utils.generate_files(
          packed_example_generator, [out_file]
      )
      generator_utils.shuffle_dataset([out_file])
  
      tf.logging.info(
          "Job stats: identity examples: {}, total examples {}, ratio: {}".format(
              self.num_identity_examples,
              self.num_total_examples,
              (1 + self.num_identity_examples)
              / (1 + self.num_total_examples),
          )
      )
  
      job_stats_string = self.aggregate_job_stats()
      out_dir, filename = out_file.replace("-unshuffled", "").rsplit(
          "/", 1
      )
      stats_prefix = "/stats_"
      stats_file_path = "".join([out_dir, stats_prefix, filename])
      if (
          tf.gfile.Exists(stats_file_path)
          and tf.gfile.Open(stats_file_path).size() != 0
      ):
          tf.logging.info(
              "Skipping writing stats because output file exists."
          )
      else:
          with tf.gfile.Open(stats_file_path, "w") as out:
              tf.logging.info(
                  "Writing job stats to {}".format(stats_file_path)
              )
              out.write(job_stats_string)
  
      tf.logging.info(job_stats_string)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import math
  import random
  from absl import flags
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.data_generators import wiki_revision_utils
  from tensor2tensor.utils import metrics
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 7 for calling the WikiRevision obj's underlying member method code with example usage: obj.generate_data(data_dir, tmp_dir, task_id) without return types