code: def test_residual_add(
      inference_module,
      batch,
      sequence,
      hidden_dim,
      dtype,
      mlp_after_attn,
      add_bias,
      mp_size,
      pre_attn_norm,
      use_triton_ops,
  ):
      if (
          not deepspeed.HAS_TRITON
          and use_triton_ops
          and dtype == torch.float16
      ):
          pytest.skip("triton has to be installed for the test")
      ds_out = torch.randn(
          (batch, sequence, hidden_dim),
          dtype=dtype,
          device=get_accelerator().device_name(),
      )
      residual = torch.randn(
          (batch, sequence, hidden_dim),
          dtype=dtype,
          device=get_accelerator().device_name(),
      )
      attn_output = torch.randn(
          (batch, sequence, hidden_dim),
          dtype=dtype,
          device=get_accelerator().device_name(),
      )
      final_bias = torch.randn(
          (hidden_dim),
          dtype=dtype,
          device=get_accelerator().device_name(),
      )
      attn_bias = torch.randn(
          (hidden_dim),
          dtype=dtype,
          device=get_accelerator().device_name(),
      )
  
      ref_out = ds_out.clone()
      ref_out = run_residual_add_reference(
          ref_out,
          residual,
          attn_output,
          attn_bias,
          final_bias,
          mlp_after_attn,
          add_bias,
          mp_size,
          pre_attn_norm,
      )
  
      res_add_args = [
          ds_out,
          residual,
          attn_output,
          attn_bias,
          final_bias,
          mp_size,
          mlp_after_attn,
          add_bias,
          pre_attn_norm,
      ]
  
      if use_triton_ops:
          from deepspeed.ops.transformer.inference.triton import (
              residual_add_bias,
          )
  
          ds_out = residual_add_bias(*res_add_args)
      if dtype == torch.float16:
          ds_out = inference_module.residual_add_bias_fp16(
              *res_add_args
          )
      elif dtype == torch.float32:
          ds_out = inference_module.residual_add_bias_fp32(
              *res_add_args
          )
      elif dtype == torch.bfloat16:
          ds_out = inference_module.residual_add_bias_bf16(
              *res_add_args
          )
      else:
          if dtype == torch.float16:
              ds_out = inference_module.residual_add_bias_fp16(
                  *res_add_args
              )
          elif dtype == torch.float32:
              ds_out = inference_module.residual_add_bias_fp32(
                  *res_add_args
              )
          else:
              raise ValueError(f"Unsupported dtype: {dtype}")
  
      if not allclose(ds_out, ref_out):
          print((ds_out - ref_out).abs().max())
          print((ds_out - ref_out).abs().mean())
          print((ds_out - ref_out))
          assert allclose(ds_out, ref_out)
  
      assert allclose(ds_out, ref_out)
  
file_path: deepspeed/tests/unit/ops/transformer/inference/test_residual_add.py
imports: import pytest
  import torch
  import deepspeed
  from deepspeed.accelerator import get_accelerator
  from deepspeed.ops.op_builder import InferenceBuilder
  from .inference_test_utils import get_dtypes
label_desc: Use image node_id 7 for calling a global function with example usage: test_residual_add(inference_module, batch, sequence, hidden_dim, dtype, mlp_after_attn, add_bias, mp_size, pre_attn_norm, use_triton_ops) without return types