class_docstr: Base class for Description2Code problems.
class_docstr_tok: ["Base","class","for","Description2Code","problems","."]
code: def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
      train = dataset_split == problem.DatasetSplit.TRAIN
      # Called twice: for train and test
  
      # Get the list of the training samples (coding challenge samples)
      samples = list(generator_samples(tmp_dir, self.pb_constants))
  
      # Split between train and dev
      # Shuffle to get problems from diverse sources (CodeChef and CodeForces) and
      # difficulties in each set.
      # Need to sort the samples first before shuffling (as walk() isn't
      # deterministic)
      samples.sort(key=lambda x: x.desc_file)  # in-place
      rng = random.Random(7531)  # Local fixed seed
      rng.shuffle(samples)  # in-place
  
      # Train: 5019/5228 problems
      # Dev: 209/5228 problems
      len_samples = len(samples)
      split = len_samples // 25
      samples = samples[split:] if train else samples[:split]
      tf.logging.info(
          "Number of samples for {}: {}/{}".format(
              "train" if train else "dev", len(samples), len_samples
          )
      )
  
      def generator_samples_content(get_source, get_target):
          """Generate samples."""
          source, target = None, None
          # Iterate over the coding samples
          for sample in samples:
              if get_source:
                  with tf.gfile.GFile(
                      sample.desc_file, mode="r"
                  ) as source_file:
                      source = source_file.read()
  
              if get_target:
                  # Each challenge can have multiple implementations (or none)
                  for code_file in sample.code_files:
                      with tf.gfile.GFile(
                          code_file, mode="r"
                      ) as target_file:
                          target = target_file.read()
                          target = self.preprocess_target(target)
                      yield source, target
              elif (
                  sample.code_files
              ):  # Only take the source if a target exists
                  yield source, target
  
      def generator_target():
          for _, target in generator_samples_content(False, True):
              yield target.strip()
  
      # Generate vocab for both source and target
  
      # TODO(lukaszkaiser): Fix vocab generation call. No sources given.
      assert not self.vocab_input_filename
      source_vocab = None
      # source_vocab = generator_utils.get_or_generate_vocab(
      #     data_dir, tmp_dir, self.vocab_input_filename, self.input_vocab_size)
  
      target_vocab = generator_utils.get_or_generate_vocab_inner(
          data_dir=data_dir,
          vocab_filename=self.vocab_target_filename,
          vocab_size=self.target_vocab_size,
          generator=generator_target(),
      )
  
      # Yield the training and testing samples
      eos_list = [EOS]
      for source, target in generator_samples_content(True, True):
          source_ints = source_vocab.encode(source.strip()) + eos_list
          target_ints = target_vocab.encode(target.strip()) + eos_list
          yield {
              "inputs": source_ints,
              "targets": target_ints,
          }
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import random
  import re
  import zipfile
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 9 for calling the Desc2CodeProblem obj's underlying member method code with example usage: obj.generate_encoded_samples(data_dir, tmp_dir, dataset_split) without return types