class_docstr: Initialize the DeepSpeed Transformer Layer.
  
  Static variable:
      layer_id: The layer-index counter starting from 0 and incrementing by 1 every time a layer object is instantiated,
      e.g. if a model has 24 transformer layers, layer_id goes from 0 to 23.
  Arguments:
      config: An object of DeepSpeedTransformerConfig
  
      initial_weights: Optional: Only used for unit test
  
      initial_biases: Optional: Only used for unit test
code: def __init__(self, config, initial_weights=None, initial_biases=None):
      super(DeepSpeedTransformerLayer, self).__init__()
  
      self.config = config
      self.config.layer_id = DeepSpeedTransformerLayer.layer_id
      DeepSpeedTransformerLayer.layer_id = (
          DeepSpeedTransformerLayer.layer_id + 1
      )
  
      print("DeepSpeed Transformer config is ", self.config.__dict__)
  
      if self.config.local_rank >= 0:
          torch.cuda.set_device(self.config.local_rank)
  
      if initial_weights is None and initial_biases is None:
          self.attn_qkvw = nn.Parameter(
              torch.Tensor(
                  self.config.hidden_size * 3, self.config.hidden_size
              )
          )
          self.attn_qkvb = nn.Parameter(
              torch.Tensor(self.config.hidden_size * 3)
          )
          self.attn_ow = nn.Parameter(
              torch.Tensor(
                  self.config.hidden_size, self.config.hidden_size
              )
          )
          self.attn_ob = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.attn_nw = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.attn_nb = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.inter_w = nn.Parameter(
              torch.Tensor(
                  self.config.intermediate_size, self.config.hidden_size
              )
          )
          self.inter_b = nn.Parameter(
              torch.Tensor(self.config.intermediate_size)
          )
          self.output_w = nn.Parameter(
              torch.Tensor(
                  self.config.hidden_size, self.config.intermediate_size
              )
          )
          self.output_b = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.norm_w = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.norm_b = nn.Parameter(
              torch.Tensor(self.config.hidden_size)
          )
          self.init_transformer_weights(self.config.adjust_init_range)
      else:
          # For testing only.
          q = initial_weights[0].data
          k = initial_weights[1].data
          v = initial_weights[2].data
  
          self.attn_qkvw = nn.Parameter(torch.cat((q, k, v)))
          # self.attn_qkvw[i * self.config.hidden_size:(i + 1) * self.config.hidden_size] = \
          #    initial_weights[i].clone()
          # torch.empty_like(initial_weights[i]).data.copy_(initial_weights[i].data)
          self.attn_qkvb = nn.Parameter(
              torch.Tensor(self.config.hidden_size * 3)
          )
          self.attn_qkvb.data.zero_()
          self.attn_ow = initial_weights[3]
          self.attn_ob = initial_biases[3]
          self.attn_nw = initial_weights[4]
          self.attn_nb = initial_biases[4]
          self.inter_w = initial_weights[5]
          self.inter_b = initial_biases[5]
          self.output_w = initial_weights[6]
          self.output_b = initial_biases[6]
          self.norm_w = initial_weights[7]
          self.norm_b = initial_biases[7]
  
      # Load cuda modules if needed
      global transformer_cuda_module, stochastic_transformer_cuda_module
      if (
          transformer_cuda_module is None
          and not self.config.stochastic_mode
      ):
          transformer_cuda_module = TransformerBuilder().load()
      if (
          stochastic_transformer_cuda_module is None
          and self.config.stochastic_mode
      ):
          stochastic_transformer_cuda_module = (
              StochasticTransformerBuilder().load()
          )
  
      # create the layer in cuda kernels.
      cuda_module = (
          stochastic_transformer_cuda_module
          if self.config.stochastic_mode
          else transformer_cuda_module
      )
      create_layer_func = (
          cuda_module.create_transformer_layer_fp16
          if self.config.fp16
          else cuda_module.create_transformer_layer_fp32
      )
  
      create_layer_func(
          self.config.layer_id,
          self.config.batch_size,
          self.config.hidden_size,
          self.config.heads,
          self.config.intermediate_size,
          self.config.attn_dropout_ratio,
          self.config.hidden_dropout_ratio,
          self.config.layer_norm_eps,
          self.config.seed,
          self.config.pre_layer_norm,
          self.config.test_gemm,
          self.config.attn_dropout_checkpoint,
          self.config.normalize_invertible,
          self.config.gelu_checkpoint,
          self.config.stochastic_mode,
      )
  
imports: import json
  import math
  import torch
  from torch import nn
  from torch.autograd import Function
  from ..op_builder import TransformerBuilder, StochasticTransformerBuilder
label_desc: Use image node_id 1 to create a new DeepSpeedTransformerLayer object from inherited base classes: nn with example: obj = DeepSpeedTransformerLayer(config, initial_weights, initial_biases)