class_docstr: PTB, 10k vocab.
class_docstr_tok: ["PTB",",","10k","vocab","."]
code: def generate_samples(self, data_dir, tmp_dir, dataset_split):
      files = _maybe_download_corpus(tmp_dir, self.vocab_type)
  
      train_file, valid_file = None, None
      for filename in files:
          if "train" in filename:
              train_file = os.path.join(tmp_dir, filename)
          elif "valid" in filename:
              valid_file = os.path.join(tmp_dir, filename)
  
      assert train_file, "Training file not found"
      assert valid_file, "Validation file not found"
  
      _get_token_encoder(data_dir, self.vocab_filename, train_file)
  
      train = dataset_split == problem.DatasetSplit.TRAIN
      filepath = train_file if train else valid_file
  
      def _generate_samples():
          with tf.gfile.GFile(filepath, "r") as f:
              for line in f:
                  line = " ".join(
                      line.replace("\n", " %s " % EOS).split()
                  )
                  yield {"targets": line}
  
      return _generate_samples()
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import sys
  import tarfile
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 5 for calling the LanguagemodelPtb10k obj's underlying member method code with example usage: obj.generate_samples(data_dir, tmp_dir, dataset_split) and returns: _generate_samples