class_docstr: Old segment -> revised segment.
  
  Data filtration heuristics:
    wiki_revision_max_page_size_exp:
      pages above this # of bytes are thrown out
  
    wiki_revision_revision_skip_factor:
      rate of logarithmic downsampling of revision history list
  
    wiki_revision_percent_identical_examples:
      how many identitcal examples to admit, as percent of total examples
  
    wiki_revision_introduce_errors:
      whether or not to introduce spelling-type errors on the source side
  
    wiki_revision_max_equal_to_diff_ratio:
      whether or not to introduce spelling-type errors on the source side
  
  
  Vocab size=32k
  Maximum input/target length = 1024 wordpiece tokens
class_docstr_tok: ["Old","segment","-",">","revised","segment",".","Data","filtration","heuristics",":","wiki_revision_max_page_size_exp",":","pages","above","this","#","of","bytes","are","thrown","out","wiki_revision_revision_skip_factor",":","rate","of","logarithmic","downsampling","of","revision","history","list","wiki_revision_percent_identical_examples",":","how","many","identitcal","examples","to","admit",",","as","percent","of","total","examples","wiki_revision_introduce_errors",":","whether","or","not","to","introduce","spelling-type","errors","on","the","source","side","wiki_revision_max_equal_to_diff_ratio",":","whether","or","not","to","introduce","spelling-type","errors","on","the","source","side","Vocab","size=32k","Maximum","input\/target","length","=","1024","wordpiece","tokens"]
code: def generator(self, encoder, corpus_files, tmp_dir):
      for page in wiki_revision_utils.corpus_page_generator(
          corpus_files, tmp_dir, FLAGS.wiki_revision_max_page_size_exp
      ):
          self.num_pages += 1
          examples = self.page_to_examples(page, encoder)
          for x in examples:
              yield x
          if self.num_total_examples % 100000 == 0:
              tf.logging.info(
                  "page count={} num_total_examples={} id={} title={}".format(
                      self.num_pages,
                      self.num_total_examples,
                      page["id"],
                      page["title"],
                  )
              )
          if (
              self.max_examples_per_shard
              and self.num_total_examples >= self.max_examples_per_shard
          ):
              tf.logging.info(
                  "Examples per shard {} >= max_examples_per_shard {}. Shutting down.".format(
                      self.num_total_examples,
                      self.max_examples_per_shard,
                  )
              )
              break
      tf.logging.info(
          "Total pages: {}, total examples: {}, examples per page: {}".format(
              self.num_pages,
              self.num_total_examples,
              0
              if not self.num_pages
              else self.num_total_examples / self.num_pages,
          )
      )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import math
  import random
  from absl import flags
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.data_generators import wiki_revision_utils
  from tensor2tensor.utils import metrics
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 8 for calling the WikiRevision obj's underlying member method code with example usage: obj.generator(encoder, corpus_files, tmp_dir) without return types