code: def forward(
      self,
      hidden_state: torch.Tensor,
      residual: torch.Tensor,
      add_bias: bool,
      attention_output: Optional[torch.Tensor] = None,
      residual_add: Optional[torch.Tensor] = None,
      attention_bias: Optional[torch.Tensor] = None,
      final_bias: Optional[torch.Tensor] = None,
  ):
      if self.residual_add_func is not None:
          if final_bias is None:
              residual = self._vector_add(
                  residual, hidden_state, 1.0 / self.config.mp_size
              )
          else:
              if (
                  not self.config.pre_layer_norm
                  and residual_add is not None
              ):
                  # only use residual add if its set and we are not pre layer norm
                  residual = residual_add
  
              self.residual_add_func(
                  hidden_state,
                  residual,
                  attention_output,
                  attention_bias,
                  final_bias,
                  self.config.mp_size,
                  self.config.mlp_after_attn,
                  add_bias,
                  self.config.pre_layer_norm,
              )
      else:
          # fallback
          if (
              os.environ.get("DS_KI_FALLBACK") == "True"
              and self.config.mlp_after_attn
          ):
              if self.config.pre_layer_norm:
                  tmp = (
                      residual.float()
                      + attention_output.float()
                      + attention_bias.float()
                      + final_bias.float()
                  ) / self.config.mp_size + hidden_state.float()
              else:
                  tmp = (
                      residual.float()
                      + hidden_state.float()
                      + final_bias.float()
                  )
  
              input_dtype = hidden_state.dtype
              residual = tmp.to(input_dtype)
          else:
              raise NotImplementedError
      return residual
  
file_path: DeepSpeed/deepspeed/ops/transformer/inference/op_binding/residual_add.py
imports: import os
  import torch
  from typing import Optional
  from ..config import DeepSpeedInferenceConfig
  from .base import BaseOp
label_desc: Use image node_id 2 for calling the ResidualAddOp obj's underlying member method code with example usage: obj.forward(hidden_state, residual, add_bias, attention_output, residual_add, attention_bias, final_bias) and returns: residual