class_docstr: Model parallelization processing engine
  
  Args:
      num_gpus (int): number of gpus
      backend (str): distributed backend (default=nccl)
      custom_policies (Union[Policy, List[Policy]]): user customized policy objects
  
  Notes:
      The parallelization process is performed through the following process.
  
      1) slice parallelizable tensors and replace original tensors on CPU
      2) upload parallelizable (replaced) tensors to multiple GPUs simultaneously
      3) upload non-parallelizable tensors to multiple GPUs (e.g. embedding, lm_head, ...)
class_docstr_tok: ["Model","parallelization","processing","engine","Args",":","num_gpus","(","int",")",":","number","of","gpus","backend","(","str",")",":","distributed","backend","(","default=nccl",")","custom_policies","(","Union","[","Policy",",","List","[","Policy","]","]",")",":","user","customized","policy","objects","Notes",":","The","parallelization","process","is","performed","through","the","following","process",".","1",")","slice","parallelizable","tensors","and","replace","original","tensors","on","CPU","2",")","upload","parallelizable","(","replaced",")","tensors","to","multiple","GPUs","simultaneously","3",")","upload","non-parallelizable","tensors","to","multiple","GPUs","(","e.g",".","embedding",",","lm_head",",","...",")"]
docstr: Create Pytorch distributed process group
  
  Args:
      backend (str): distributed backend
  
  Returns:
      ProcessGroupNCCL: process group for parallization
docstr_tok: ["Create","Pytorch","distributed","process","group","Args",":","backend","(","str",")",":","distributed","backend","Returns",":","ProcessGroupNCCL",":","process","group","for","parallization"]
code: def create_process_group(self, backend: str):
      """
      Create Pytorch distributed process group
  
      Args:
          backend (str): distributed backend
  
      Returns:
          ProcessGroupNCCL: process group for parallization
      """
      if not dist.is_initialized():
          dist.init_process_group(backend=backend)
  
      torch.cuda.set_device(int(os.getenv("LOCAL_RANK", "0")))
      new_group = dist.new_group([i for i in range(self.num_gpus)])
  
      return new_group
  
file_path: parallelformers/parallelformers/parallel/engine.py
imports: import os
  from typing import List, Union
  import torch
  import torch.distributed
  import torch.nn
  from parallelformers.parallel.replacing import TensorReplacer
  from parallelformers.policies.base import Policy
  from parallelformers.utils import rsetattr
label_desc: Use image node_id 3 for calling the ParallelEngine obj's underlying member method code with example usage: obj.create_process_group(backend) and returns: new_group