class_docstr: Replace original Huggingface's layer into Megatron tensor sliced layer.
  
  Args:
      model (nn.Module): Huggingface pre-trained transformer model
      mp_group (Any): process group for model parallelism
      fp16: (bool): Whether use FP16 or not.
      num_gpus (int): number of GPUs
      custom_policies (Union[Policy, List[Policy]]): custom policy object (default=None)
class_docstr_tok: ["Replace","original","Huggingface","'s","layer","into","Megatron","tensor","sliced","layer",".","Args",":","model","(","nn.Module",")",":","Huggingface","pre-trained","transformer","model","mp_group","(","Any",")",":","process","group","for","model","parallelism","fp16",":","(","bool",")",":","Whether","use","FP16","or","not",".","num_gpus","(","int",")",":","number","of","GPUs","custom_policies","(","Union","[","Policy",",","List","[","Policy","]","]",")",":","custom","policy","object","(","default=None",")"]
docstr: Replace modules in the model by user defined policy
  
  Args:
      model (nn.Module): model weight
      policy_cls (Type[Policy]): class of policy
docstr_tok: ["Replace","modules","in","the","model","by","user","defined","policy","Args",":","model","(","nn.Module",")",":","model","weight","policy_cls","(","Type","[","Policy","]",")",":","class","of","policy"]
code: def replace_user_define_modules(
      self,
      model: nn.Module,
      policy_cls: Type[Policy],
  ) -> None:
      """
      Replace modules in the model by user defined policy
  
      Args:
          model (nn.Module): model weight
          policy_cls (Type[Policy]): class of policy
      """
      for _, child in model.named_children():
          if child.__class__ == nn.ModuleList:
              child = child[0]
  
          replace_modules = policy_cls.replace_modules()
  
          if child.__class__.__qualname__ in replace_modules.keys():
              for cls_name, cls in replace_modules.items():
                  if child.__class__.__qualname__ == cls_name:
                      for key in cls.__dict__.keys():
                          rsetattr(
                              child.__class__,
                              "mp_group",
                              self.mp_group,
                          )
  
                          if rhasattr(child.__class__, key):
                              rsetattr(
                                  child.__class__,
                                  key,
                                  rgetattr(cls, key),
                              )
  
          self.replace_user_define_modules(child, policy_cls)
  
file_path: parallelformers/parallelformers/parallel/replacing.py
imports: from contextlib import suppress
  from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
  import torch.nn
  from torch import Size, Tensor
  from parallelformers.parallel.slicing import TensorSlicer
  from parallelformers.policies.base import Layer, Policy
  from parallelformers.policies.base.auto import AutoPolicy
  from parallelformers.utils import rgetattr, rhasattr, rsetattr
label_desc: Use image node_id 4 for calling the TensorReplacer obj's underlying member method code with example usage: obj.replace_user_define_modules(model, policy_cls) without return types