class_docstr: ### LoRA weights
  G = G + Ag @ Bg
  U = U + Au @ Bu
  W = W + Aw @ Bw
  
  ### SwiGLU(X)
  e = X @ G
  f = e * sigmoid(e)
  g = X @ U
  h = f * g
  i = h @ W
  
  ### Backpropagation chain rule
  See our blog post for more details
  
  df = sigmoid(e) * (1 - f) + f
  dC/dW = h.T @ dY
  dC/dU = X.T @ (D @ W.T * f)
  dC/dG = X.T @ (D @ W.T * df * g)
  
  ### Down projection LoRA weights
  dC/dAw = dC/dW @ B.T
  dC/dBw = A.T @ dC/dW
  dC/dAw =       h.T @ dY @ B.T
  dC/dBw = A.T @ h.T @ dY
  
  ### Up projection LoRA weights
  dC/dAu =       X.T @ (D @ W.T * f) @ B.T
  dC/dBu = A.T @ X.T @ (D @ W.T * f)
  
  ### Gate projection LoRA weights
  dC/dAg =       X.T @ (D @ W.T * df * g) @ B.T
  dC/dBg = A.T @ X.T @ (D @ W.T * df * g)
  
  Don't forget to see our blog post for more details!
class_docstr_tok: ["#","#","#","LoRA","weights","G","=","G","+","Ag","@","Bg","U","=","U","+","Au","@","Bu","W","=","W","+","Aw","@","Bw","#","#","#","SwiGLU","(","X",")","e","=","X","@","G","f","=","e","*","sigmoid","(","e",")","g","=","X","@","U","h","=","f","*","g","i","=","h","@","W","#","#","#","Backpropagation","chain","rule","See","our","blog","post","for","more","details","df","=","sigmoid","(","e",")","*","(","1","-","f",")","+","f","dC\/dW","=","h.T","@","dY","dC\/dU","=","X.T","@","(","D","@","W.T","*","f",")","dC\/dG","=","X.T","@","(","D","@","W.T","*","df","*","g",")","#","#","#","Down","projection","LoRA","weights","dC\/dAw","=","dC\/dW","@","B.T","dC\/dBw","=","A.T","@","dC\/dW","dC\/dAw","=","h.T","@","dY","@","B.T","dC\/dBw","=","A.T","@","h.T","@","dY","#","#","#","Up","projection","LoRA","weights","dC\/dAu","=","X.T","@","(","D","@","W.T","*","f",")","@","B.T","dC\/dBu","=","A.T","@","X.T","@","(","D","@","W.T","*","f",")","#","#","#","Gate","projection","LoRA","weights","dC\/dAg","=","X.T","@","(","D","@","W.T","*","df","*","g",")","@","B.T","dC\/dBg","=","A.T","@","X.T","@","(","D","@","W.T","*","df","*","g",")","Do","n't","forget","to","see","our","blog","post","for","more","details","!"]
code: def forward(
      ctx,
      X: torch.Tensor,
      gateW,
      gateW_quant,
      gateA,
      gateB,
      gateS,
      upW,
      upW_quant,
      upA,
      upB,
      upS,
      downW,
      downW_quant,
      downA,
      downB,
      downS,
  ):
      dtype = X.dtype
  
      e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)
      g = matmul_lora(X, upW, upW_quant, upA, upB, upS)
      h = swiglu_fg_kernel(e, g)
      i = matmul_lora(h, downW, downW_quant, downA, downB, downS)
  
      ctx.custom_saved_tensors = (
          gateW,
          gateW_quant,
          gateS,
          upW,
          upW_quant,
          upS,
          downW,
          downW_quant,
          downS,
      )
      ctx.save_for_backward(
          gateA, gateB, upA, upB, downA, downB, X, e, g
      )
      return i
  
file_path: unsloth/unsloth/kernels/fast_lora.py
imports: import torch
  from .utils import fast_dequantize, QUANT_STATE
  from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
label_desc: Use image node_id 1 for calling the LoRA_MLP obj's underlying member method code with example usage: obj.forward(ctx, X, gateW, gateW_quant, gateA, gateB, gateS, upW, upW_quant, upA, upB, upS, downW, downW_quant, downA, downB, downS) and returns: i