code: def __init__(
      self,
      inference=True,
      linear_layer=True,
      scale_attention=True,
      megatron_v2=False,
      use_mup=False,
      # the type of activation function used in MLP
      mlp_act_func_type=ActivationFuncType.GELU,
      # applies layer norm before attention if `pre_attn_norm` is set to True
      pre_attn_norm=True,
      # this flag shows whether or not using prefix in loading the checkpoint
      use_load_prefix=False,
      # whether or not the qkv is stored in the split-format
      split_qkv=True,
      # Type of normalization to perform
      norm_type=NormType.LayerNorm,
  ):
      super().__init__()
      self.cuda_graph_supported = False
      self.inference = inference
      self.linear_layer = linear_layer
      self.scale_attention = scale_attention
      self.is_megatron_v2 = megatron_v2
      self.use_mup = use_mup
      self.mlp_act_func_type = mlp_act_func_type
      self.pre_attn_norm = pre_attn_norm
      self.use_load_prefix = use_load_prefix
      self.split_qkv = split_qkv
      self.norm_type = norm_type
  
imports: from abc import ABC, abstractmethod
  from deepspeed.utils.types import ActivationFuncType, NormType
  import torch
  from deepspeed.accelerator import get_accelerator
label_desc: Use image node_id 1 to create a new TransformerPolicy object from inherited base classes: DSPolicy with example: obj = TransformerPolicy(inference, linear_layer, scale_attention, megatron_v2, use_mup, mlp_act_func_type, pre_attn_norm, use_load_prefix, split_qkv, norm_type)