class_docstr: A schedule for training a batch using hybrid parallelism.
  
  Pipeline parallelism is extracted through gradient accumulation and thus
  convergence follows that of a data parallel approach with the same batch
  size.
class_docstr_tok: ["A","schedule","for","training","a","batch","using","hybrid","parallelism",".","Pipeline","parallelism","is","extracted","through","gradient","accumulation","and","thus","convergence","follows","that","of","a","data","parallel","approach","with","the","same","batch","size","."]
code: def steps(self):
      """"""
      prev_micro_batch_id = -1
      total_steps = 2 * (self.micro_batches + self.stages - 1)
      for step_id in range(total_steps):
          # Map the step of the pipeline to the micro-batch id and also whether it is a
          # forward or backward pass step.
          micro_batch_id, is_forward = self._step_to_micro_batch(
              step_id
          )
  
          if self._valid_micro_batch(prev_micro_batch_id):
              prev_buffer = self._buffer_idx(prev_micro_batch_id)
          if self._valid_micro_batch(micro_batch_id):
              curr_buffer = self._buffer_idx(micro_batch_id)
  
          cmds = []
  
          # Exchange activations
          if is_forward:
              if self._valid_micro_batch(
                  prev_micro_batch_id
              ) and self._valid_stage(self.prev_stage):
                  cmds.append(SendGrad(prev_buffer))
              if self._valid_micro_batch(
                  micro_batch_id
              ) and self._valid_stage(self.prev_stage):
                  cmds.append(RecvActivation(curr_buffer))
          else:
              if self._valid_micro_batch(
                  micro_batch_id
              ) and self._valid_stage(self.next_stage):
                  cmds.append(RecvGrad(curr_buffer))
              if self._valid_micro_batch(
                  prev_micro_batch_id
              ) and self._valid_stage(self.next_stage):
                  cmds.append(SendActivation(prev_buffer))
  
          # First/last stage loads
          if self.stage_id == 0 or self.stage_id == self.stages - 1:
              if is_forward and self._valid_micro_batch(micro_batch_id):
                  cmds.append(LoadMicroBatch(curr_buffer))
  
          # Computation
          if self._valid_micro_batch(micro_batch_id):
              if is_forward:
                  cmds.append(ForwardPass(curr_buffer))
              else:
                  cmds.append(BackwardPass(curr_buffer))
  
          # Model step at the end of the batch
          if step_id == total_steps - 1:
              cmds.append(ReduceTiedGrads())
              cmds.append(ReduceGrads())
              cmds.append(OptimizerStep())
  
          # Prepare state for next time
          prev_micro_batch_id = micro_batch_id
          yield cmds
  
imports: from ..utils import call_to_str
  from abc import ABC, abstractmethod
label_desc: Use image node_id 1 for calling the TrainSchedule obj's underlying member method code with example usage: obj.steps() without return types