docstr: weights need be a state_dict of bert model
docstr_tok: ["weights","need","be","a","state_dict","of","bert","model"]
code: def __init__(
      self,
      layer_num,
      hidden_dim,
      weights=None,
      sparse=False,
      tensor_para_size=1,
      pipeline_para_size=1,
  ):
      """weights need be a state_dict of bert model"""
      self.layer_num = layer_num
      self.int8 = False
      self.hidden_dim = hidden_dim
      self.weights = {}
      self.tensor_para_size = tensor_para_size
      self.pipeline_para_size = pipeline_para_size
  
      self.use_mpi = dist.is_mpi_available()
  
      if self.use_mpi:
          try:
              dist.init_process_group(backend="mpi")
          except:  # noqa: E722
              logger.info(
                  "[INFO] WARNING: Exception occurred in "
                  "dist.init_process_group(backend='mpi')."
                  "Maybe the process group has been initialized somewhere else."  # noqa: E501
              )
      else:
          logger.info(
              "[INFO] MPI is not available in this PyTorch build."
          )
          assert (
              tensor_para_size == 1
          ), "[FATAL] MPI is required for tensor_para_size > 1."
          assert (
              pipeline_para_size == 1
          ), "[FATAL] MPI is required for pipeline_para_size > 1."
  
      self.rank = dist.get_rank() if self.use_mpi else 0
      self.device_count = torch.cuda.device_count()
      self.device = self.rank % self.device_count
      torch.cuda.set_device(self.device)
  
      world_size = (
          dist.get_world_size() if self.use_mpi else 1
      )  # noqa: F841
      self.tensor_para_rank = self.rank % self.tensor_para_size
      self.pipeline_para_rank = self.rank // self.tensor_para_size
      if weights is None:
          self._generated_weights = True
          for i in range(layer_num):
              pre = "encoder.layer." + str(i) + "."
              self.weights[
                  pre + "attention.self.query.weight"
              ] = torch.zeros(hidden_dim, hidden_dim)
              self.weights[
                  pre + "attention.self.query.bias"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "attention.self.key.weight"
              ] = torch.zeros(hidden_dim, hidden_dim)
              self.weights[
                  pre + "attention.self.key.bias"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "attention.self.value.weight"
              ] = torch.zeros(hidden_dim, hidden_dim)
              self.weights[
                  pre + "attention.self.value.bias"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "attention.output.dense.weight"
              ] = torch.zeros(hidden_dim, hidden_dim)
              self.weights[
                  pre + "attention.output.dense.bias"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "attention.output.LayerNorm.weight"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "attention.output.LayerNorm.bias"
              ] = torch.zeros(hidden_dim)
              self.weights[
                  pre + "intermediate.dense.weight"
              ] = torch.zeros(
                  4 * hidden_dim, hidden_dim
              )  # noqa: E501
              self.weights[
                  pre + "intermediate.dense.bias"
              ] = torch.zeros(4 * hidden_dim)
              self.weights[pre + "output.dense.weight"] = torch.zeros(
                  hidden_dim, 4 * hidden_dim
              )
              self.weights[pre + "output.dense.bias"] = torch.zeros(
                  hidden_dim
              )
              self.weights[
                  pre + "output.LayerNorm.weight"
              ] = torch.zeros(hidden_dim)
              self.weights[pre + "output.LayerNorm.bias"] = torch.zeros(
                  hidden_dim
              )
          for k, v in self.weights.items():
              if not k.endswith("_amax"):
                  self.weights[k] = torch.nn.init.uniform_(v, -1, 1)
          if sparse:
              for k, v in self.weights.items():
                  if (
                      "query.weight" in k
                      or "key.weight" in k
                      or "value.weight" in k
                      or "dense.weight" in k
                  ):
                      v_shape = v.shape
                      v = v.view(-1, 4)
                      _, indices = torch.topk(
                          torch.abs(v), 2, dim=-1, largest=False
                      )
                      v.scatter_(1, indices, 0)
                      self.weights[k] = v.view(v_shape)
      else:
          self._generated_weights = False
          for k, v in weights.items():
              ks = k.split(".")
              if ks[-2] == "LayerNorm":
                  if ks[-1] == "gamma":
                      ks[-1] = "weight"
                  elif ks[-1] == "beta":
                      ks[-1] = "bias"
              self.weights[".".join(ks)] = v
  
imports: from typing import List, Optional
  from loguru import logger
  from nebullvm.optional_modules.torch import torch, torch_distributed
  from nebullvm.optional_modules.huggingface import BertConfig, BertEmbeddings, BertEncoder, BertPooler, BertPreTrainedModel
  from .checkpoint_quantization import checkpoint_quantization
label_desc: Use image node_id 1 to create a new EncoderWeights object from inherited base classes: object with example: obj = EncoderWeights(layer_num, hidden_dim, weights, sparse, tensor_para_size, pipeline_para_size)