code: def reference_geglu_implementation(
      input: torch.Tensor,
      bias: Optional[torch.Tensor] = None,
      act_fn: Optional[ActivationType] = ActivationType.GEGLU,
  ) -> torch.Tensor:
      act_func_map = {
          ActivationType.ReGLU: torch.nn.functional.relu,
          ActivationType.GEGLU: lambda x: torch.nn.functional.gelu(
              x, approximate="tanh"
          ),
          ActivationType.SiGLU: torch.nn.functional.silu,
      }
  
      dtype = input.dtype
      input = input.to(torch.float32)
  
      if bias is not None:
          bias = bias.to(torch.float32)
          input = input + bias
  
      act_act = input[..., ::2]
      act_linear = input[..., 1::2]
  
      act_act = act_func_map[act_fn](act_act)
  
      return (act_act * act_linear).to(dtype)
  
file_path: DeepSpeed/tests/unit/inference/v2/kernels/core_ops/test_gated_activation.py
imports: from typing import Iterable, Optional
  import pytest
  import torch
  from deepspeed.accelerator import get_accelerator
  from deepspeed.inference.v2.kernels.core_ops import CUDAGatedActivation
  from deepspeed.inference.v2.inference_utils import ActivationType
  from ....v2.inference_test_utils import get_dtypes, allclose
label_desc: Use image node_id 1 for calling a global function with example usage: reference_geglu_implementation(input, bias, act_fn) and returns: unknown