class_docstr: An object that slices tensors into rows or columns as described in the Megatron LM paper
  
  Args:
      mp_group (torch.distributed.ProcessGroupNCCL): Distributed group for model parallelism
docstr: Slice tensors into rows or columns as described in the Megatron LM paper
  
  Args:
      tensor (Dict): tensor dictionaries
      attributes (Dict): attributes dictionaries
      dim (int): dimension for slicing
      is_bias (bool): whether tensor is bias or not
  
  Returns:
      Tuple: tuple of sliced tensors
docstr_tok: ["Slice","tensors","into","rows","or","columns","as","described","in","the","Megatron","LM","paper","Args",":","tensor","(","Dict",")",":","tensor","dictionaries","attributes","(","Dict",")",":","attributes","dictionaries","dim","(","int",")",":","dimension","for","slicing","is_bias","(","bool",")",":","whether","tensor","is","bias","or","not","Returns",":","Tuple",":","tuple","of","sliced","tensors"]
code: def slice_tensor(
      self,
      tensor: Dict,
      attributes: Dict,
      dim: int,
      is_bias: bool,
  ) -> Tuple:
      """
      Slice tensors into rows or columns as described in the Megatron LM paper
  
      Args:
          tensor (Dict): tensor dictionaries
          attributes (Dict): attributes dictionaries
          dim (int): dimension for slicing
          is_bias (bool): whether tensor is bias or not
  
      Returns:
          Tuple: tuple of sliced tensors
      """
      if not tensor:
          return (None,)
  
      n_fused_list, reversed_list = [], []
      for (k_tensor, _), (k_attr, v_attr) in zip(
          tensor.items(),
          attributes.items(),
      ):
          if k_tensor == k_attr:
              n_fused, reversed = v_attr
              n_fused_list.append(n_fused)
              reversed_list.append(reversed)
  
      tensor = list(tensor.values())
      slices = []
  
      for proj_layer, n_fused, reversed in zip(
          tensor,
          n_fused_list,
          reversed_list,
      ):
          device = torch.cuda.current_device()
          dim = dim if not reversed or is_bias else abs(dim - 1)
          n_fused = 1 if not n_fused else n_fused
  
          proj_layer = proj_layer.chunk(
              n_fused * self.world_size,
              dim=dim,
          )
  
          if n_fused > 1:
              ranks = (
                  len(proj_layer) + self.world_size - 1
              ) // self.world_size
              proj_layer = [
                  proj_layer[
                      i * self.world_size : (i + 1) * self.world_size
                  ]
                  for i in range(ranks)
              ]
              proj_layer = list(
                  map(
                      lambda x: torch.cat([*x], dim=-1),
                      zip(*proj_layer),
                  )
              )
  
          proj_layer = proj_layer[self.gpu_index].to(device)
          slices.append(proj_layer)
  
      return tuple(slices)
  
file_path: parallelformers/parallelformers/parallel/slicing.py
imports: import os
  from typing import Dict, Tuple
  import torch
  import torch.distributed
label_desc: Use image node_id 2 for calling the TensorSlicer obj's underlying member method code with example usage: obj.slice_tensor(tensor, attributes, dim, is_bias) and returns: tuple, None