class_docstr: Linear layer with row parallelism.
  
  The linear layer is defined as Y = XA + b. A is parallelized along its first dimension and X along its second dimension. A = transpose([A_1 .. A_p]) X = [X_1, ..., X_p]
  
  Args:
      input_size: first dimension of matrix A.
      output_size: second dimension of matrix A.
      bias: If true, add bias. Note that bias is not parallelized.
      input_is_parallel: If true, we assume that the input is already split across the GPUs and we do not split again.
      init_method: method to initialize weights. Note that bias is always set to zero.
      stride: For the strided linear layers.
      keep_master_weight_for_test: This was added for testing and should be set to False. It returns the master weights used for initialization.
      skip_bias_add: If True, do not add the bias term, instead return it to be added by the caller. This enables performance optimations where bias can be fused with other elementwise operations.
      is_expert: If True, the layer is treated as an MoE expert layer
      tp_comm_buffer_name: Communication buffer name. Not used in
                           non-Transformer-Engine modules.
      config: ModelParallelConfig object
code: def __init__(
      self,
      input_size: int,
      output_size: int,
      *,
      config: ModelParallelConfig,
      init_method: Callable,
      bias: bool,
      input_is_parallel: bool,
      skip_bias_add: bool,
      stride: int = 1,
      keep_master_weight_for_test: bool = False,
      is_expert: bool = False,
      tp_comm_buffer_name: str = None,  # Not used
  ):
      super(RowParallelLinear, self).__init__()
  
      # Keep input parameters
      self.input_size = input_size
      self.output_size = output_size
      self.input_is_parallel = input_is_parallel
      # Divide the weight matrix along the last dimension.
      world_size = get_tensor_model_parallel_world_size()
      self.input_size_per_partition = divide(input_size, world_size)
      self.skip_bias_add = skip_bias_add
      self.config = config
      self.is_expert = is_expert
      self.expert_parallel = config.expert_model_parallel_size > 1
      self.gradient_accumulation_fusion = (
          config.gradient_accumulation_fusion
      )
      self.sequence_parallel = config.sequence_parallel
      if self.sequence_parallel and not self.input_is_parallel:
          raise RuntimeError(
              "To enable `sequence_parallel`, `input_is_parallel` must be `True`"
          )
  
      # Parameters.
      # Note: torch.nn.functional.linear performs XA^T + b and as a result
      # we allocate the transpose.
      # Initialize weight.
      if config.use_cpu_initialization:
          self.weight = Parameter(
              torch.empty(
                  self.output_size,
                  self.input_size_per_partition,
                  dtype=config.params_dtype,
              )
          )
          if config.perform_initialization:
              self.master_weight = _initialize_affine_weight_cpu(
                  self.weight,
                  self.output_size,
                  self.input_size,
                  self.input_size_per_partition,
                  1,
                  init_method,
                  stride=stride,
                  return_master_weight=keep_master_weight_for_test,
                  params_dtype=config.params_dtype,
              )
      else:
          self.weight = Parameter(
              torch.empty(
                  self.output_size,
                  self.input_size_per_partition,
                  device=torch.cuda.current_device(),
                  dtype=config.params_dtype,
              )
          )
          if config.perform_initialization:
              _initialize_affine_weight_gpu(
                  self.weight,
                  init_method,
                  partition_dim=1,
                  stride=stride,
                  expert_parallel=(
                      self.is_expert and self.expert_parallel
                  ),
              )
      setattr(
          self.weight,
          "allreduce",
          not (self.is_expert and self.expert_parallel),
      )
  
      if bias:
          if config.use_cpu_initialization:
              self.bias = Parameter(
                  torch.empty(
                      self.output_size, dtype=config.params_dtype
                  )
              )
          else:
              self.bias = Parameter(
                  torch.empty(
                      self.output_size,
                      device=torch.cuda.current_device(),
                      dtype=config.params_dtype,
                  )
              )
  
          if config.perform_initialization:
              # Always initialize bias to zero.
              with torch.no_grad():
                  self.bias.zero_()
          setattr(
              self.bias,
              "allreduce",
              not (self.is_expert and self.expert_parallel),
          )
          setattr(
              self.bias, "sequence_parallel", self.sequence_parallel
          )
      else:
          self.register_parameter("bias", None)
  
      self._forward_impl = (
          linear_with_grad_accumulation_and_async_allreduce
      )
      self.explicit_expert_comm = self.is_expert and (
          self.sequence_parallel or self.expert_parallel
      )
  
imports: import math
  import os
  import warnings
  from typing import Callable, Optional
  import torch
  import torch.nn.functional
  import torch.nn.init
  from torch.cuda.amp import custom_bwd, custom_fwd
  from torch.nn.parameter import Parameter
  from megatron.core.model_parallel_config import ModelParallelConfig
  from megatron.core.parallel_state import get_global_memory_buffer, get_tensor_model_parallel_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size
  from .mappings import copy_to_tensor_model_parallel_region, gather_from_sequence_parallel_region, gather_from_tensor_model_parallel_region, reduce_from_tensor_model_parallel_region, reduce_scatter_to_sequence_parallel_region, scatter_to_tensor_model_parallel_region
  from .random import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name
  from .utils import VocabUtility, divide, split_tensor_along_last_dim
label_desc: Use image node_id 1 to create a new RowParallelLinear object from inherited base classes: torch.nn with example: obj = RowParallelLinear(input_size, output_size)