class_docstr: Base Problem for gene expression datasets.
class_docstr_tok: ["Base","Problem","for","gene","expression","datasets","."]
code: def generate_data(self, data_dir, tmp_dir, task_id=-1):
      try:
          # Download source data if download_url specified
          h5_filepath = generator_utils.maybe_download(
              tmp_dir, self.h5_file, self.download_url
          )
      except NotImplementedError:
          # Otherwise, look for it locally
          h5_filepath = os.path.join(tmp_dir, self.h5_file)
  
      with h5py.File(h5_filepath, "r") as h5_file:
          num_train_examples = h5_file["train_in"].len()
          num_dev_examples = h5_file["valid_in"].len()
          num_test_examples = h5_file["test_in"].len()
  
      # Collect all_filepaths to later shuffle
      all_filepaths = []
      # Collect created shard processes to start and join
      processes = []
  
      datasets = [
          (
              self.training_filepaths,
              self.num_shards,
              "train",
              num_train_examples,
          ),
          (self.dev_filepaths, 10, "valid", num_dev_examples),
          (self.test_filepaths, 10, "test", num_test_examples),
      ]
      for fname_fn, nshards, key_prefix, num_examples in datasets:
          outfiles = fname_fn(data_dir, nshards, shuffled=False)
          all_filepaths.extend(outfiles)
          for start_idx, end_idx, outfile in generate_shard_args(
              outfiles, num_examples
          ):
              p = mp.Process(
                  target=generate_dataset,
                  args=(
                      h5_filepath,
                      key_prefix,
                      [outfile],
                      self.chunk_size,
                      start_idx,
                      end_idx,
                  ),
              )
              processes.append(p)
  
      # 1 per training shard + 10 for dev + 10 for test
      assert len(processes) == self.num_shards + 20
  
      # Start and wait for processes in batches
      num_batches = int(
          math.ceil(float(len(processes)) / MAX_CONCURRENT_PROCESSES)
      )
      for i in range(num_batches):
          start = i * MAX_CONCURRENT_PROCESSES
          end = start + MAX_CONCURRENT_PROCESSES
          current = processes[start:end]
          for p in current:
              p.start()
          for p in current:
              p.join()
  
      # Shuffle
      generator_utils.shuffle_dataset(all_filepaths)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import math
  import multiprocessing
  import os
  import h5py
  import numpy
  from six.moves import range
  from tensor2tensor.data_generators import dna_encoder
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.layers import modalities
  from tensor2tensor.utils import metrics
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 7 for calling the GeneExpressionProblem obj's underlying member method code with example usage: obj.generate_data(data_dir, tmp_dir, task_id) without return types