class_docstr: Model parallelization processing engine
  
  Args:
      num_gpus (int): number of gpus
      backend (str): distributed backend (default=nccl)
      custom_policies (Union[Policy, List[Policy]]): user customized policy objects
  
  Notes:
      The parallelization process is performed through the following process.
  
      1) slice parallelizable tensors and replace original tensors on CPU
      2) upload parallelizable (replaced) tensors to multiple GPUs simultaneously
      3) upload non-parallelizable tensors to multiple GPUs (e.g. embedding, lm_head, ...)
class_docstr_tok: ["Model","parallelization","processing","engine","Args",":","num_gpus","(","int",")",":","number","of","gpus","backend","(","str",")",":","distributed","backend","(","default=nccl",")","custom_policies","(","Union","[","Policy",",","List","[","Policy","]","]",")",":","user","customized","policy","objects","Notes",":","The","parallelization","process","is","performed","through","the","following","process",".","1",")","slice","parallelizable","tensors","and","replace","original","tensors","on","CPU","2",")","upload","parallelizable","(","replaced",")","tensors","to","multiple","GPUs","simultaneously","3",")","upload","non-parallelizable","tensors","to","multiple","GPUs","(","e.g",".","embedding",",","lm_head",",","...",")"]
code: def __init__(
      self,
      num_gpus: int,
      backend: str,
      custom_policies: Union[Policy, List[Policy]],
  ) -> None:
      self.num_gpus = num_gpus
      self.custom_policies = custom_policies
      self.mp_group = self.create_process_group(backend)
  
file_path: parallelformers/parallelformers/parallel/engine.py
imports: import os
  from typing import List, Union
  import torch
  import torch.distributed
  import torch.nn
  from parallelformers.parallel.replacing import TensorReplacer
  from parallelformers.policies.base import Policy
  from parallelformers.utils import rsetattr
label_desc: Use image node_id 1 to create a new ParallelEngine object from inherited base classes: object with example: obj = ParallelEngine(num_gpus, backend, custom_policies)