class_docstr: Linear layer with row parallelism.
  
  The linear layer is defined as Y = XA + b. A is parallelized along
  its first dimension and X along its second dimension as:
             -   -
            | A_1 |
            | .   |
        A = | .   |        X = [X_1, ..., X_p]
            | .   |
            | A_p |
             -   -
  
  .. note::
      Input is supposed to be three dimensional and each dimension
      is expected to be sequence, batch, and hidden feature, respectively.
  
  Arguments:
      input_size: first dimension of matrix A.
      output_size: second dimension of matrix A.
      bias: If true, add bias. Note that bias is not parallelized.
      input_is_parallel: If true, we assume that the input is already
                         split across the GPUs and we do not split
                         again.
      init_method: method to initialize weights. Note that bias is always set
                   to zero.
      stride: For the strided linear layers.
      keep_master_weight_for_test: This was added for testing and should be
                                   set to False. It returns the master weights
                                   used for initialization.
      skip_bias_add: This was added to enable performance optimization where bias
                     can be fused with other elementwise operations. We skip
                     adding bias but instead return it.
  Keyword Arguments:
      params_dtype:
      use_cpu_initialization:
      gradient_accumulation_fusion:
      sequence_parallel_enabled:
      accumulation_in_fp16: Deprecated
code: def __init__(
      self,
      input_size,
      output_size,
      bias=True,
      input_is_parallel=False,
      init_method=init.xavier_normal_,
      stride=1,
      keep_master_weight_for_test=False,
      skip_bias_add=False,
      *,
      params_dtype=torch.float32,
      use_cpu_initialization=False,
      gradient_accumulation_fusion=False,
      sequence_parallel_enabled: bool = False,
      accumulation_in_fp16: Optional[bool] = None,
  ):
      super().__init__()
  
      # Keep input parameters
      self.input_size = input_size
      self.output_size = output_size
      self.input_is_parallel = input_is_parallel
      # Divide the weight matrix along the last dimension.
      world_size = get_tensor_model_parallel_world_size()
      self.input_size_per_partition = divide(input_size, world_size)
      self.skip_bias_add = skip_bias_add
      self.gradient_accumulation_fusion = gradient_accumulation_fusion
      self.sequence_parallel_enabled = sequence_parallel_enabled
      if self.sequence_parallel_enabled and not self.input_is_parallel:
          raise RuntimeError(
              "To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`"
          )
  
      if accumulation_in_fp16 is not None:
          warnings.warn(
              f"Deprecated option `accumulation_in_fp16` is set to {accumulation_in_fp16}"
          )
  
      # as an argument to this function?
      # Parameters.
      # Note: torch.nn.functional.linear performs XA^T + b and as a result
      # we allocate the transpose.
      # Initialize weight.
      if use_cpu_initialization:
          self.weight = Parameter(
              torch.empty(
                  self.output_size,
                  self.input_size_per_partition,
                  dtype=params_dtype,
              )
          )
          self.master_weight = _initialize_affine_weight_cpu(
              self.weight,
              self.output_size,
              self.input_size,
              self.input_size_per_partition,
              1,
              init_method,
              stride=stride,
              return_master_weight=keep_master_weight_for_test,
              params_dtype=params_dtype,
          )
      else:
          self.weight = Parameter(
              torch.empty(
                  self.output_size,
                  self.input_size_per_partition,
                  device=torch.cuda.current_device(),
                  dtype=params_dtype,
              )
          )
          _initialize_affine_weight_gpu(
              self.weight, init_method, partition_dim=1, stride=stride
          )
      if bias:
          if use_cpu_initialization:
              self.bias = Parameter(
                  torch.empty(self.output_size, dtype=params_dtype)
              )
          else:
              self.bias = Parameter(
                  torch.empty(
                      self.output_size,
                      device=torch.cuda.current_device(),
                      dtype=params_dtype,
                  )
              )
          # Always initialize bias to zero.
          with torch.no_grad():
              self.bias.zero_()
          setattr(
              self.bias,
              "sequence_parallel_enabled",
              sequence_parallel_enabled,
          )
      else:
          self.register_parameter("bias", None)
  
      self._forward_impl = (
          linear_with_grad_accumulation_and_async_allreduce
      )
  
imports: from typing import Optional, Dict, Tuple, List
  import warnings
  import torch
  import torch.nn.functional
  import torch.nn.init
  from torch.nn.parameter import Parameter
  from apex._autocast_utils import _cast_if_autocast_enabled
  from apex.transformer.parallel_state import get_tensor_model_parallel_group
  from apex.transformer.parallel_state import get_tensor_model_parallel_rank
  from apex.transformer.parallel_state import get_tensor_model_parallel_world_size
  from apex.transformer.utils import divide
  from apex.transformer.tensor_parallel.mappings import copy_to_tensor_model_parallel_region
  from apex.transformer.tensor_parallel.mappings import gather_from_tensor_model_parallel_region
  from apex.transformer.tensor_parallel.mappings import reduce_from_tensor_model_parallel_region
  from apex.transformer.tensor_parallel.mappings import scatter_to_tensor_model_parallel_region
  from apex.transformer.tensor_parallel.mappings import reduce_scatter_to_sequence_parallel_region
  from apex.transformer.tensor_parallel.random import get_cuda_rng_tracker
  from apex.transformer.tensor_parallel.utils import VocabUtility
  from apex.transformer.log_util import get_transformer_logger
label_desc: Use image node_id 1 to create a new RowParallelLinear object from inherited base classes: torch.nn with example: obj = RowParallelLinear(input_size, output_size, bias, input_is_parallel, init_method, stride, keep_master_weight_for_test, skip_bias_add)