class_docstr: Parallel GatedMlp
class_docstr_tok: ["Parallel","GatedMlp"]
code: def __init__(
      self,
      in_features,
      process_group,
      hidden_features=None,
      out_features=None,
      activation=F.sigmoid,
      bias1=True,
      bias2=True,
      multiple_of=128,
      sequence_parallel=True,
      device=None,
      dtype=None,
  ):
      factory_kwargs = {"device": device, "dtype": dtype}
      super().__init__()
      out_features = (
          out_features if out_features is not None else in_features
      )
      hidden_features = (
          hidden_features
          if hidden_features is not None
          else int(8 * in_features / 3)
      )
      hidden_features = (
          (hidden_features + multiple_of - 1)
          // multiple_of
          * multiple_of
      )
      if ColumnParallelLinear is None or RowParallelLinear is None:
          raise ImportError("fused_dense is not installed")
      self.fc1 = ColumnParallelLinear(
          in_features,
          2 * hidden_features,
          process_group,
          bias=bias1,
          sequence_parallel=sequence_parallel,
          **factory_kwargs,
      )
      self.activation = activation
      self.fc2 = RowParallelLinear(
          hidden_features,
          out_features,
          process_group,
          bias=bias2,
          sequence_parallel=sequence_parallel,
          **factory_kwargs,
      )
  
imports: import torch
  import torch.nn
  import torch.nn.functional
  from torch.distributed import ProcessGroup
label_desc: Use image node_id 1 to create a new ParallelGatedMlp object from inherited base classes: nn with example: obj = ParallelGatedMlp(in_features, process_group, hidden_features, out_features, activation, bias1, bias2, multiple_of, sequence_parallel, device, dtype)