class_docstr: Fused MLP layer for incrementally improved training efficiency
class_docstr_tok: ["Fused","MLP","layer","for","incrementally","improved","training","efficiency"]
code: def __init__(
      self,
      config,
      gate_proj: torch.nn.Linear,
      up_proj: torch.nn.Linear,
      down_proj: torch.nn.Linear,
  ):
      super().__init__()
      self.config = config
      self.swiglu = SwiGLU(
          in_features=config.hidden_size,
          hidden_features=config.intermediate_size,
          bias=False,
          _pack_weights=True,
      )
      # overwrite initialized weights with pretrained weights
      self.swiglu.w12.weight.data = torch.cat(
          (gate_proj.weight.data, up_proj.weight.data), dim=0
      )
      self.swiglu.w3.weight.data = down_proj.weight.data
  
imports: import logging
  import warnings
  from functools import partial
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn.functional
  import transformers
  from einops import rearrange
  from flash_attn.bert_padding import pad_input, unpad_input
  from transformers.modeling_outputs import BaseModelOutputWithPast
  from transformers.models.llama.modeling_llama import LlamaAttention
  from transformers.models.llama.modeling_llama import LlamaDecoderLayer
  from transformers.models.llama.modeling_llama import LlamaMLP, apply_rotary_pos_emb, repeat_kv
  from xformers.ops import SwiGLU
  from axolotl.monkeypatch.utils import get_cu_seqlens_from_pos_ids, set_module_name
label_desc: Use image node_id 1 to create a new FusedMLP object from inherited base classes: torch.nn with example: obj = FusedMLP(config, gate_proj, up_proj, down_proj)