class_docstr: Enwiki8, with examples up to 65,536 characters long.
class_docstr_tok: ["Enwiki8",",","with","examples","up","to","65,536","characters","long","."]
code: def generate_samples(self, data_dir, tmp_dir, dataset_split):
      filepath = _maybe_download_corpus(tmp_dir)
      with tf.io.gfile.GFile(filepath, mode=self.READ_MODE) as f:
          data = f.read()
  
      tf.logging.info("Length of enwik8 = %d", len(data))
  
      num_test_chars = 5000000
  
      if dataset_split == problem.DatasetSplit.TRAIN:
          part = data[: -2 * num_test_chars]
      elif dataset_split == problem.DatasetSplit.EVAL:
          part = data[-2 * num_test_chars : -num_test_chars]
      elif dataset_split == problem.DatasetSplit.TEST:
          part = data[-num_test_chars:]
      else:
          raise ValueError("Undefined dataset_split")
  
      tf.logging.info(
          "Length of split '%s' = %d", dataset_split, len(part)
      )
  
      # TODO(kitaev): Better handling of evaluation data, to ensure that there is
      # always context available.
      if dataset_split == problem.DatasetSplit.TRAIN:
          offset = self.sequence_length // self.DUPE_FACTOR
          for start in range(0, len(part), offset):
              yield {
                  "targets": part[start : start + self.sequence_length]
              }
      else:
          for start in range(0, len(part), self.sequence_length):
              yield {
                  "targets": part[start : start + self.sequence_length]
              }
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import os
  import zipfile
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 7 for calling the Enwik8L65k obj's underlying member method code with example usage: obj.generate_samples(data_dir, tmp_dir, dataset_split) without return types