code: def __build_slurm_command(
      self, trial, slurm_cmd_script_path, timestamp, exp_i, on_gpu
  ):
      sub_commands = []
  
      command = [
          "#!/bin/bash",
          "#",
          "# Auto-generated by test-tube (https://github.com/williamFalcon/test-tube)",
          "#################\n",
      ]
      sub_commands.extend(command)
  
      # add job name
      job_with_version = "{}v{}".format(self.job_display_name, exp_i)
      command = [
          "# set a job name",
          "#SBATCH --job-name={}".format(job_with_version),
          "#################\n",
      ]
      sub_commands.extend(command)
  
      # add out output
      if self.enable_log_out:
          out_path = os.path.join(
              self.out_log_path,
              "{}_slurm_output_%j.out".format(timestamp),
          )
          command = [
              "# a file for job output, you can check job progress",
              "#SBATCH --output={}".format(out_path),
              "#################\n",
          ]
          sub_commands.extend(command)
  
      # add err output
      if self.enable_log_err:
          err_path = os.path.join(
              self.err_log_path,
              "{}_slurm_output_%j.err".format(timestamp),
          )
          command = [
              "# a file for errors",
              "#SBATCH --error={}".format(err_path),
              "#################\n",
          ]
          sub_commands.extend(command)
  
      # add job time
      command = [
          "# time needed for job",
          "#SBATCH --time={}".format(self.job_time),
          "#################\n",
      ]
      sub_commands.extend(command)
  
      # add nb of gpus
      if self.per_experiment_nb_gpus > 0 and on_gpu:
          command = [
              "# gpus per node",
              "#SBATCH --gres=gpu:{}".format(
                  self.per_experiment_nb_gpus
              ),
              "#################\n",
          ]
          if self.gpu_type is not None:
              command = [
                  "# gpus per node",
                  "#SBATCH --gres=gpu:{}:{}".format(
                      self.gpu_type, self.per_experiment_nb_gpus
                  ),
                  "#################\n",
              ]
          sub_commands.extend(command)
  
      # add nb of cpus if not looking at a gpu job
      if self.per_experiment_nb_cpus > 0:
          command = [
              "# cpus per job",
              "#SBATCH --cpus-per-task={}".format(
                  self.per_experiment_nb_cpus
              ),
              "#################\n",
          ]
          sub_commands.extend(command)
  
      # pick nb nodes
      command = [
          "# number of requested nodes",
          "#SBATCH --nodes={}".format(self.per_experiment_nb_nodes),
          "#################\n",
      ]
      sub_commands.extend(command)
  
      # pick memory per node
      command = [
          "# memory per node",
          "#SBATCH --mem={}".format(self.memory_mb_per_node),
          "#################\n",
      ]
      sub_commands.extend(command)
  
      # add signal command to catch job termination
      command = [
          "# slurm will send a signal this far out before it kills the job",
          f"#SBATCH --signal=USR1@{self.minutes_to_checkpoint_before_walltime * 60}",
          "#################\n",
      ]
  
      sub_commands.extend(command)
  
      # Subscribe to email if requested
      mail_type = []
      if self.notify_on_end:
          mail_type.append("END")
      if self.notify_on_fail:
          mail_type.append("FAIL")
      if len(mail_type) > 0:
          mail_type_query = [
              "# Have SLURM send you an email when the job ends or fails",
              "#SBATCH --mail-type={}".format(",".join(mail_type)),
          ]
          sub_commands.extend(mail_type_query)
  
          email_query = [
              "#SBATCH --mail-user={}".format(self.email),
          ]
          sub_commands.extend(email_query)
  
      # add custom sbatch commands
      sub_commands.append("\n")
      for cmd, value, comment in self.slurm_commands:
          comment = "# {}".format(comment)
          cmd = "#SBATCH --{}={}".format(cmd, value)
          spaces = "#################\n"
          sub_commands.extend([comment, cmd, spaces])
  
      # load modules
      sub_commands.append("\n")
      for module in self.modules:
          cmd = "module load {}".format(module)
          sub_commands.append(cmd)
  
      # remove spaces before the hash
      sub_commands = [x.lstrip() for x in sub_commands]
  
      # add additional commands
      for cmd in self.commands:
          sub_commands.append(cmd)
          sub_commands.append("\n")
  
      # add run command
      trial_args = self.__get_hopt_params(trial)
      trial_args = "{} --{} {} --{} {}".format(
          trial_args,
          HyperOptArgumentParser.SLURM_CMD_PATH,
          slurm_cmd_script_path,
          HyperOptArgumentParser.SLURM_EXP_CMD,
          exp_i,
      )
  
      cmd = "srun {} {} {}".format(
          self.python_cmd, self.script_name, trial_args
      )
      sub_commands.append(cmd)
  
      # build full command with empty lines in between
      full_command = "\n".join(sub_commands)
      return full_command
  
imports: import datetime
  import os
  import signal
  import sys
  import time
  import traceback
  from subprocess import call
  from .argparse_hopt import HyperOptArgumentParser
label_desc: Use image node_id 16 for calling the SlurmCluster obj's underlying member method code with example usage: obj.__build_slurm_command(trial, slurm_cmd_script_path, timestamp, exp_i, on_gpu) and returns: full_command