class_docstr: A single transformer layer.
  Transformer layer takes input with size [b, s, h] and returns an
  output of the same size.
code: def __init__(
      self,
      layer_number,
      hidden_size,
      num_attention_heads,
      attention_dropout,
      mlp_ratio,
      hidden_dropout,
      is_naive_fp16,
      apply_residual_connection_post_layernorm=False,
      fp32_residual_connection=False,
      bias_dropout_fusion: bool = True,
      convert_fp16_to_fp32_in_softmax: bool = False,
  ):
      super().__init__()
      self.layer_number = layer_number
  
      self.apply_residual_connection_post_layernorm = (
          apply_residual_connection_post_layernorm
      )
      self.fp32_residual_connection = fp32_residual_connection
  
      # Layernorm on the input data.
      self.input_layernorm = LayerNorm(hidden_size)
  
      # Self attention.
      self.self_attention = TransformerSelfAttentionRing(
          hidden_size=hidden_size,
          num_attention_heads=num_attention_heads,
          attention_dropout=attention_dropout,
          attention_mask_func=attention_mask_func,
          layer_number=layer_number,
          apply_query_key_layer_scaling=True,
          convert_fp16_to_fp32_in_softmax=convert_fp16_to_fp32_in_softmax,
          fp16=is_naive_fp16,
      )
  
      self.hidden_dropout = hidden_dropout
      self.bias_dropout_fusion = bias_dropout_fusion
  
      # Layernorm on the attention output
      self.post_attention_layernorm = LayerNorm(hidden_size)
  
      self.mlp = TransformerMLP(
          hidden_size=hidden_size, mlp_ratio=mlp_ratio
      )
  
file_path: colossalai/examples/tutorial/sequence_parallel/model/layers/bert_layer.py
imports: import torch
  import torch.nn
  from colossalai.kernel.cuda_native import LayerNorm
  from colossalai.kernel.jit import bias_dropout_add_fused_inference, bias_dropout_add_fused_train
  from colossalai.legacy.nn.layer.parallel_sequence import TransformerSelfAttentionRing
  from .dropout import get_bias_dropout_add
  from .mlp import TransformerMLP
label_desc: Use image node_id 1 to create a new BertLayer object from inherited base classes: nn with example: obj = BertLayer(layer_number, hidden_size, num_attention_heads, attention_dropout, mlp_ratio, hidden_dropout, is_naive_fp16, apply_residual_connection_post_layernorm, fp32_residual_connection, bias_dropout_fusion, convert_fp16_to_fp32_in_softmax)