code: def __init__(
      self,
      in_features: int,
      out_features: int,
      process_group: ProcessGroup,
      bias: bool = True,
      sequence_parallel=True,
      multiple_of=1,
      device=None,
      dtype=None,
  ) -> None:
      world_size = torch.distributed.get_world_size(process_group)
      rank = torch.distributed.get_rank(process_group)
      if in_features % multiple_of:
          raise ValueError(
              f"in_features ({in_features}) must be a multiple of {multiple_of}"
          )
      multiple = in_features // multiple_of
      # We want to split @multiple across world_size, but it could be an uneven split
      div = multiple // world_size
      mod = multiple % world_size
      # The first @mod ranks get @div + 1 copies, the rest get @div copies
      local_multiple = div + int(
          torch.distributed.get_rank(process_group) < mod
      )
      # Only rank 0 will have bias
      super().__init__(
          local_multiple * multiple_of,
          out_features,
          bias=bias and rank == 0,
          device=device,
          dtype=dtype,
      )
      self.process_group = process_group
      self.sequence_parallel = sequence_parallel
  
imports: from functools import partial
  from typing import Optional
  import fused_dense_lib
  import torch
  import torch.nn
  import torch.nn.functional
  from torch import Tensor
  from torch.cuda.amp import custom_bwd, custom_fwd
  from torch.distributed import ProcessGroup
  from flash_attn.ops.activations import gelu_bwd, relu_bwd, sqrelu_bwd, sqrelu_fwd
  from flash_attn.utils.distributed import all_gather_raw, all_reduce, all_reduce_raw, reduce_scatter, reduce_scatter_raw
label_desc: Use image node_id 1 to create a new RowParallelLinear object from inherited base classes: nn with example: obj = RowParallelLinear(in_features, out_features, process_group, bias, sequence_parallel, multiple_of, device, dtype)