code: def get_or_generate_vocabulary(
      data_dir,
      tmp_dir,
      data_prefix,
      max_page_size_exp,
      approx_vocab_size=32768,
      strip=True,
  ):
      """Get or generate the vocabulary.
  
      Args:
        data_dir: a string
        tmp_dir: a string
        data_prefix: a string
        max_page_size_exp: an integer
        approx_vocab_size: an integer
        strip: a boolean
  
      Returns:
        a TextEncoder
      """
      num_pages_for_vocab_generation = approx_vocab_size // 3
      vocab_file = vocab_filename(approx_vocab_size, strip)
  
      def my_generator(data_prefix):
          """Line generator for vocab."""
          count = 0
          for page in corpus_page_generator(
              all_corpus_files(data_prefix)[::-1],
              tmp_dir,
              max_page_size_exp,
          ):
              revisions = page["revisions"]
              if revisions:
                  text = get_text(revisions[-1], strip=strip)
                  yield text
                  count += 1
                  if count % 100 == 0:
                      tf.logging.info(
                          "reading pages for vocab %d" % count
                      )
                  if count > num_pages_for_vocab_generation:
                      break
  
      return generator_utils.get_or_generate_vocab_inner(
          data_dir,
          vocab_file,
          approx_vocab_size,
          my_generator(data_prefix),
      )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import math
  import os
  import random
  import re
  import subprocess
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import text_encoder
  import tensorflow.compat.v1
label_desc: Use image node_id 21 for calling a global function with example usage: get_or_generate_vocabulary(data_dir, tmp_dir, data_prefix, max_page_size_exp, approx_vocab_size, strip) and returns: generator_utils