class_docstr: Class that controls the training process.
  
  Args:
      model(:py:class:`onmt.models.model.NMTModel`): model to train
      train_loss(:obj:`onmt.utils.loss.LossComputeBase`):
        training loss computation
      valid_loss(:obj:`onmt.utils.loss.LossComputeBase`):
        training loss computation
      scoring_preparator(:obj:`onmt.translate.utils.ScoringPreparator`):
        preparator for the calculation of metrics via the
        _eval_handler method
      valid_scorers (dict): keeps in memory the current values
        of the validation metrics
      optim(:obj:`onmt.utils.optimizers.Optimizer`):
        the optimizer responsible for update
      trunc_size(int): length of truncated back propagation
        through time
      accum_count(list): accumulate gradients this many times.
      accum_steps(list): steps for accum gradients changes.
      n_gpu (int): number of gpu.
      gpu_rank (int): ordinal rank of the gpu in the list.
      report_manager(:obj:`onmt.utils.ReportMgrBase`):
        the object that creates reports, or None
      with_align (bool): whether to jointly lear alignment
        (Transformer)
      model_saver(:obj:`onmt.models.ModelSaverBase`): the saver is
        used to save a checkpoint.
        Thus nothing will be saved if this parameter is None.
      average_decay (float): cf opt.average_decay
      average_every (int): average model every x steps.
      model_dtype (str): fp32 or fp16.
      earlystopper (:obj:`onmt.utils.EarlyStopping`): add early
        stopping mecanism
      dropout (float): dropout value in RNN or FF layers.
      attention_dropout (float): dropaout in attention layers.
      dropout_steps (list): dropout values scheduling in steps.
      zero_out_prompt_loss (bool): whether to zero-out the prompt loss
          (mostly for LLM finetuning).
class_docstr_tok: ["Class","that","controls","the","training","process",".","Args",":","model","(",":","py",":","class",":","`","onmt.models.model.NMTModel","`",")",":","model","to","train","train_loss","(",":","obj",":","`","onmt.utils.loss.LossComputeBase","`",")",":","training","loss","computation","valid_loss","(",":","obj",":","`","onmt.utils.loss.LossComputeBase","`",")",":","training","loss","computation","scoring_preparator","(",":","obj",":","`","onmt.translate.utils.ScoringPreparator","`",")",":","preparator","for","the","calculation","of","metrics","via","the","_eval_handler","method","valid_scorers","(","dict",")",":","keeps","in","memory","the","current","values","of","the","validation","metrics","optim","(",":","obj",":","`","onmt.utils.optimizers.Optimizer","`",")",":","the","optimizer","responsible","for","update","trunc_size","(","int",")",":","length","of","truncated","back","propagation","through","time","accum_count","(","list",")",":","accumulate","gradients","this","many","times",".","accum_steps","(","list",")",":","steps","for","accum","gradients","changes",".","n_gpu","(","int",")",":","number","of","gpu",".","gpu_rank","(","int",")",":","ordinal","rank","of","the","gpu","in","the","list",".","report_manager","(",":","obj",":","`","onmt.utils.ReportMgrBase","`",")",":","the","object","that","creates","reports",",","or","None","with_align","(","bool",")",":","whether","to","jointly","lear","alignment","(","Transformer",")","model_saver","(",":","obj",":","`","onmt.models.ModelSaverBase","`",")",":","the","saver","is","used","to","save","a","checkpoint",".","Thus","nothing","will","be","saved","if","this","parameter","is","None",".","average_decay","(","float",")",":","cf","opt.average_decay","average_every","(","int",")",":","average","model","every","x","steps",".","model_dtype","(","str",")",":","fp32","or","fp16",".","earlystopper","(",":","obj",":","`","onmt.utils.EarlyStopping","`",")",":","add","early","stopping","mecanism","dropout","(","float",")",":","dropout","value","in","RNN","or","FF","layers",".","attention_dropout","(","float",")",":","dropaout","in","attention","layers",".","dropout_steps","(","list",")",":","dropout","values","scheduling","in","steps",".","zero_out_prompt_loss","(","bool",")",":","whether","to","zero-out","the","prompt","loss","(","mostly","for","LLM","finetuning",")","."]
docstr: Function that iterates over big batches = ``true_batches``
  
  Perform a backward on the loss of each sub_batch and
  finally update the params at the end of the big batch.
docstr_tok: ["Function","that","iterates","over","big","batches","=","``","true_batches","``","Perform","a","backward","on","the","loss","of","each","sub_batch","and","finally","update","the","params","at","the","end","of","the","big","batch","."]
code: def _gradient_accumulation(
      self, true_batches, normalization, total_stats, report_stats
  ):
      """Function that iterates over big batches = ``true_batches``
  
      Perform a backward on the loss of each sub_batch and
      finally update the params at the end of the big batch."""
  
      if self.accum_count > 1:
          self.optim.zero_grad(set_to_none=True)
  
      for k, batch in enumerate(true_batches):
          target_size = batch["tgt"].size(1)
          # Truncated BPTT: reminder not compatible with accum > 1
          if self.trunc_size:
              trunc_size = self.trunc_size
          else:
              trunc_size = target_size
  
          src = batch["src"]
          src_len = batch["srclen"]
          if src_len is not None:
              report_stats.n_src_words += src_len.sum().item()
              total_stats.n_src_words += src_len.sum().item()
  
          tgt_outer = batch["tgt"]
  
          bptt = False
          for j in range(0, target_size - 1, trunc_size):
              # 1. Create truncated target.
  
              tgt = tgt_outer[:, j : j + trunc_size, :]
  
              # 2. F-prop all but generator.
              if self.accum_count == 1:
                  self.optim.zero_grad(set_to_none=True)
              try:
                  with torch.cuda.amp.autocast(enabled=self.optim.amp):
                      model_out, attns = self.model(
                          src,
                          tgt,
                          src_len,
                          bptt=bptt,
                          with_align=self.with_align,
                      )
                      bptt = True
  
                      # 3. Compute loss.
                      if self.zero_out_prompt_loss:
                          # The loss of the prompt will be set to zero.
                          batch = self.train_loss.ignore_prompt(batch)
                      loss, batch_stats = self.train_loss(
                          batch,
                          model_out,
                          attns,
                          trunc_start=j,
                          trunc_size=trunc_size,
                      )
                  if loss is not None:
                      loss /= normalization
                      self.optim.backward(loss)
  
                  total_stats.update(batch_stats)
                  report_stats.update(batch_stats)
  
              except Exception as exc:
                  trace_content = traceback.format_exc()
                  if "CUDA out of memory" in trace_content:
                      logger.info(
                          "Step %d, cuda OOM - batch removed",
                          self.optim.training_step,
                      )
                      torch.cuda.empty_cache()
                      if (
                          self.n_gpu > 1
                          and self.parallel_mode == "tensor_parallel"
                      ):
                          torch.distributed.destroy_process_group()
                          sys.exit()
                  else:
                      traceback.print_exc()
                      raise exc
  
              # If truncated, don't backprop fully.
              if self.model.decoder.state != {}:
                  self.model.decoder.detach_state()
  
      # in case of multi step gradient accumulation,
      # update only after accum batches
      if self.n_gpu > 1 and self.parallel_mode == "data_parallel":
          grads = [
              p.grad.data
              for p in self.model.parameters()
              if p.requires_grad and p.grad is not None
          ]
          onmt.utils.distributed.all_reduce_and_rescale_tensors(
              grads, float(self.n_gpu)
          )
  
      self.optim.step()
  
imports: import time
  import sys
  import torch
  import traceback
  import onmt.utils
  from onmt.utils.loss import LossCompute
  from onmt.utils.logging import logger
  from onmt.utils.scoring_utils import ScoringPreparator
  from onmt.scorers import get_scorers_cls, build_scorers
label_desc: Use image node_id 9 for calling the Trainer obj's underlying member method code with example usage: obj._gradient_accumulation(true_batches, normalization, total_stats, report_stats) without return types