class_docstr: A language model on the 1B words corpus.
  
  Ratio of dev tokens (including eos) to dev words (including eos)
  176923 / 159658 = 1.108137; multiply log_ppl by this to compare results.
class_docstr_tok: ["A","language","model","on","the","1B","words","corpus",".","Ratio","of","dev","tokens","(","including","eos",")","to","dev","words","(","including","eos",")","176923","\/","159658","=","1.108137",";","multiply","log_ppl","by","this","to","compare","results","."]
code: def generate_samples(self, data_dir, tmp_dir, dataset_split):
      del data_dir
      split_files = {
          problem.DatasetSplit.TRAIN: _train_data_filenames(tmp_dir),
          problem.DatasetSplit.EVAL: _dev_data_filenames(tmp_dir),
      }
      _maybe_download_corpus(tmp_dir)
      original_vocab = _original_vocab(tmp_dir)
      files = split_files[dataset_split]
      for filepath in files:
          tf.logging.info("filepath = %s", filepath)
          for line in tf.gfile.Open(filepath):
              txt = _replace_oov(
                  original_vocab, text_encoder.native_to_unicode(line)
              )
              yield {"targets": txt}
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import os
  import tarfile
  from six.moves import range
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 4 for calling the LanguagemodelLm1b32k obj's underlying member method code with example usage: obj.generate_samples(data_dir, tmp_dir, dataset_split) without return types