class_docstr: Initialize the MultiHeadAttention.
  
  Static variable:
  
      layer_id: The layer-index counter starting from 0 and incrementing by 1 every time a layer object is instantiated,
      e.g. if a model has 24 transformer layers, layer_id goes from 0 to 23.
  
  Arguments:
      hidden_size: Total dimension of hidden_size.
      nhead: Number of parallel attention heads.
      batch_size: Batch Size for one forward
      max_seq_len: Max length of input sequence
      dropout: Dropout probability
      norm_first: perform LayerNorms before attention
class_docstr_tok: ["Initialize","the","MultiHeadAttention",".","Static","variable",":","layer_id",":","The","layer-index","counter","starting","from","0","and","incrementing","by","1","every","time","a","layer","object","is","instantiated",",","e.g",".","if","a","model","has","24","transformer","layers",",","layer_id","goes","from","0","to","23",".","Arguments",":","hidden_size",":","Total","dimension","of","hidden_size",".","nhead",":","Number","of","parallel","attention","heads",".","batch_size",":","Batch","Size","for","one","forward","max_seq_len",":","Max","length","of","input","sequence","dropout",":","Dropout","probability","norm_first",":","perform","LayerNorms","before","attention"]
code: def calc_bound(self, w):
      fan_in, _ = nn.init._calculate_fan_in_and_fan_out(w)
      bound = 1.0 / math.sqrt(fan_in)
      return bound
  
file_path: ColossalAI/colossalai/kernel/cuda_native/multihead_attention.py
imports: import math
  from dataclasses import dataclass
  import torch
  from torch import nn
  from torch.autograd import Function
label_desc: Use image node_id 2 for calling the MultiHeadAttention obj's underlying member method code with example usage: obj.calc_bound(w) and returns: bound