code: def train_new_models(
      dir,
      iter,
      srand,
      num_jobs,
      num_archives_processed,
      num_archives,
      raw_model_string,
      egs_dir,
      momentum,
      max_param_change,
      shuffle_buffer_size,
      minibatch_size_str,
      image_augmentation_opts,
      run_opts,
      frames_per_eg=-1,
      min_deriv_time=None,
      max_deriv_time_relative=None,
      use_multitask_egs=False,
      train_opts="",
      backstitch_training_scale=0.0,
      backstitch_training_interval=1,
  ):
      """Called from train_one_iteration(), this model does one iteration of
      training with 'num_jobs' jobs, and writes files like
      exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw
  
      We cannot easily use a single parallel SGE job to do the main training,
      because the computation of which archive and which --frame option
      to use for each job is a little complex, so we spawn each one separately.
      this is no longer true for RNNs as we use do not use the --frame option
      but we use the same script for consistency with FF-DNN code
  
      Selected args:
          frames_per_eg:
              The frames_per_eg, in the context of (non-chain) nnet3 training,
              is normally the number of output (supervised) frames in each training
              example.  However, the frames_per_eg argument to this function should
              only be set to that number (greater than zero) if you intend to
              train on a single frame of each example, on each minibatch.  If you
              provide this argument >0, then for each training job a different
              frame from the dumped example is selected to train on, based on
              the option --frame=n to nnet3-copy-egs.
              If you leave frames_per_eg at its default value (-1), then the
              entire sequence of frames is used for supervision.  This is suitable
              for RNN training, where it helps to amortize the cost of computing
              the activations for the frames of context needed for the recurrence.
          use_multitask_egs : True, if different examples used to train multiple
              tasks or outputs, e.g.multilingual training.  multilingual egs can
              be generated using get_egs.sh and
              steps/nnet3/multilingual/allocate_multilingual_examples.py, those
              are the top-level scripts.
      """
  
      chunk_level_training = False if frames_per_eg > 0 else True
  
      deriv_time_opts = []
      if min_deriv_time is not None:
          deriv_time_opts.append(
              "--optimization.min-deriv-time={0}".format(min_deriv_time)
          )
      if max_deriv_time_relative is not None:
          deriv_time_opts.append(
              "--optimization.max-deriv-time-relative={0}".format(
                  max_deriv_time_relative
              )
          )
  
      threads = []
  
      # the GPU timing info is only printed if we use the --verbose=1 flag; this
      # slows down the computation slightly, so don't accumulate it on every
      # iteration.  Don't do it on iteration 0 either, because we use a smaller
      # than normal minibatch size, and people may get confused thinking it's
      # slower for iteration 0 because of the verbose option.
      verbose_opt = "--verbose=1" if iter % 20 == 0 and iter > 0 else ""
  
      for job in range(1, num_jobs + 1):
          # k is a zero-based index that we will derive the other indexes from.
          k = num_archives_processed + job - 1
  
          # work out the 1-based archive index.
          archive_index = (k % num_archives) + 1
  
          if not chunk_level_training:
              frame = (
                  k // num_archives + archive_index
              ) % frames_per_eg
  
          cache_io_opts = (
              "--read-cache={dir}/cache.{iter}".format(
                  dir=dir, iter=iter
              )
              if iter > 0
              else ""
          ) + (
              " --write-cache={0}/cache.{1}".format(dir, iter + 1)
              if job == 1
              else ""
          )
  
          if image_augmentation_opts:
              image_augmentation_cmd = "nnet3-egs-augment-image --srand={srand} {aug_opts} ark:- ark:- |".format(
                  srand=k + srand, aug_opts=image_augmentation_opts
              )
          else:
              image_augmentation_cmd = ""
  
          multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
              egs_dir,
              egs_prefix="egs.",
              archive_index=archive_index,
              use_multitask_egs=use_multitask_egs,
          )
  
          scp_or_ark = "scp" if use_multitask_egs else "ark"
  
          egs_rspecifier = """ark,bg:nnet3-copy-egs {frame_opts} {multitask_egs_opts} \
              {scp_or_ark}:{egs_dir}/egs.{archive_index}.{scp_or_ark} ark:- | \
              nnet3-shuffle-egs --buffer-size={shuffle_buffer_size} \
              --srand={srand} ark:- ark:- | {aug_cmd} \
              nnet3-merge-egs --minibatch-size={minibatch_size} ark:- ark:- |""".format(
              frame_opts=(
                  ""
                  if chunk_level_training
                  else "--frame={0}".format(frame)
              ),
              egs_dir=egs_dir,
              archive_index=archive_index,
              shuffle_buffer_size=shuffle_buffer_size,
              minibatch_size=minibatch_size_str,
              aug_cmd=image_augmentation_cmd,
              srand=iter + srand,
              scp_or_ark=scp_or_ark,
              multitask_egs_opts=multitask_egs_opts,
          )
  
          # note: the thread waits on that process's completion.
          thread = common_lib.background_command(
              """{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \
                      nnet3-train {parallel_train_opts} {cache_io_opts} \
                       {verbose_opt} --print-interval=10 \
                      --momentum={momentum} \
                      --max-param-change={max_param_change} \
                      --backstitch-training-scale={backstitch_training_scale} \
                      --l2-regularize-factor={l2_regularize_factor} \
                      --backstitch-training-interval={backstitch_training_interval} \
                      --srand={srand} {train_opts} \
                      {deriv_time_opts} "{raw_model}" "{egs_rspecifier}" \
                      {dir}/{next_iter}.{job}.raw""".format(
                  command=run_opts.command,
                  train_queue_opt=run_opts.train_queue_opt,
                  dir=dir,
                  iter=iter,
                  next_iter=iter + 1,
                  srand=iter + srand,
                  job=job,
                  parallel_train_opts=run_opts.parallel_train_opts,
                  cache_io_opts=cache_io_opts,
                  verbose_opt=verbose_opt,
                  momentum=momentum,
                  max_param_change=max_param_change,
                  l2_regularize_factor=1.0 / num_jobs,
                  backstitch_training_scale=backstitch_training_scale,
                  backstitch_training_interval=backstitch_training_interval,
                  train_opts=train_opts,
                  deriv_time_opts=" ".join(deriv_time_opts),
                  raw_model=raw_model_string,
                  egs_rspecifier=egs_rspecifier,
              ),
              require_zero_status=True,
          )
  
          threads.append(thread)
  
      for thread in threads:
          thread.join()
  
imports: from __future__ import print_function
  from __future__ import division
  import glob
  import logging
  import math
  import os
  import random
  import time
  import libs.common
  import libs.nnet3.train.common
label_desc: Use image node_id 1 for calling a global function with example usage: train_new_models(dir, iter, srand, num_jobs, num_archives_processed, num_archives, raw_model_string, egs_dir, momentum, max_param_change, shuffle_buffer_size, minibatch_size_str, image_augmentation_opts, run_opts, frames_per_eg, min_deriv_time, max_deriv_time_relative, use_multitask_egs, train_opts, backstitch_training_scale, backstitch_training_interval) without return types