class_docstr: Replace original Huggingface's layer into Megatron tensor sliced layer.
  
  Args:
      model (nn.Module): Huggingface pre-trained transformer model
      mp_group (Any): process group for model parallelism
      fp16: (bool): Whether use FP16 or not.
      num_gpus (int): number of GPUs
      custom_policies (Union[Policy, List[Policy]]): custom policy object (default=None)
class_docstr_tok: ["Replace","original","Huggingface","'s","layer","into","Megatron","tensor","sliced","layer",".","Args",":","model","(","nn.Module",")",":","Huggingface","pre-trained","transformer","model","mp_group","(","Any",")",":","process","group","for","model","parallelism","fp16",":","(","bool",")",":","Whether","use","FP16","or","not",".","num_gpus","(","int",")",":","number","of","GPUs","custom_policies","(","Union","[","Policy",",","List","[","Policy","]","]",")",":","custom","policy","object","(","default=None",")"]
code: def __init__(
      self,
      model: nn.Module,
      mp_group: Any,
      fp16: bool,
      num_gpus: int,
      custom_policies: Union[Policy, List[Policy]],
  ) -> None:
      self.model = model
      self.config = model.config
      self.fp16 = fp16
      self.num_gpus = num_gpus
      self.mp_group = mp_group
      self.varname = "layer"
      self.slicer = TensorSlicer(self.mp_group)
  
      if isinstance(custom_policies, Iterable):
          self.policies = custom_policies
      elif isinstance(custom_policies, Policy):
          self.policies = [custom_policies]
      else:
          self.policies = None
  
file_path: parallelformers/parallelformers/parallel/replacing.py
imports: from contextlib import suppress
  from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
  import torch.nn
  from torch import Size, Tensor
  from parallelformers.parallel.slicing import TensorSlicer
  from parallelformers.policies.base import Layer, Policy
  from parallelformers.policies.base.auto import AutoPolicy
  from parallelformers.utils import rgetattr, rhasattr, rsetattr
label_desc: Use image node_id 1 to create a new TensorReplacer object from inherited base classes: object with example: obj = TensorReplacer(model, mp_group, fp16, num_gpus, custom_policies)