class_docstr: ### LoRA weights
  G = G + Ag @ Bg
  U = U + Au @ Bu
  W = W + Aw @ Bw
  
  ### SwiGLU(X)
  e = X @ G
  f = e * sigmoid(e)
  g = X @ U
  h = f * g
  i = h @ W
  
  ### Backpropagation chain rule
  See our blog post for more details
  
  df = sigmoid(e) * (1 - f) + f
  dC/dW = h.T @ dY
  dC/dU = X.T @ (D @ W.T * f)
  dC/dG = X.T @ (D @ W.T * df * g)
  
  ### Down projection LoRA weights
  dC/dAw = dC/dW @ B.T
  dC/dBw = A.T @ dC/dW
  dC/dAw =       h.T @ dY @ B.T
  dC/dBw = A.T @ h.T @ dY
  
  ### Up projection LoRA weights
  dC/dAu =       X.T @ (D @ W.T * f) @ B.T
  dC/dBu = A.T @ X.T @ (D @ W.T * f)
  
  ### Gate projection LoRA weights
  dC/dAg =       X.T @ (D @ W.T * df * g) @ B.T
  dC/dBg = A.T @ X.T @ (D @ W.T * df * g)
  
  Don't forget to see our blog post for more details!
class_docstr_tok: ["#","#","#","LoRA","weights","G","=","G","+","Ag","@","Bg","U","=","U","+","Au","@","Bu","W","=","W","+","Aw","@","Bw","#","#","#","SwiGLU","(","X",")","e","=","X","@","G","f","=","e","*","sigmoid","(","e",")","g","=","X","@","U","h","=","f","*","g","i","=","h","@","W","#","#","#","Backpropagation","chain","rule","See","our","blog","post","for","more","details","df","=","sigmoid","(","e",")","*","(","1","-","f",")","+","f","dC\/dW","=","h.T","@","dY","dC\/dU","=","X.T","@","(","D","@","W.T","*","f",")","dC\/dG","=","X.T","@","(","D","@","W.T","*","df","*","g",")","#","#","#","Down","projection","LoRA","weights","dC\/dAw","=","dC\/dW","@","B.T","dC\/dBw","=","A.T","@","dC\/dW","dC\/dAw","=","h.T","@","dY","@","B.T","dC\/dBw","=","A.T","@","h.T","@","dY","#","#","#","Up","projection","LoRA","weights","dC\/dAu","=","X.T","@","(","D","@","W.T","*","f",")","@","B.T","dC\/dBu","=","A.T","@","X.T","@","(","D","@","W.T","*","f",")","#","#","#","Gate","projection","LoRA","weights","dC\/dAg","=","X.T","@","(","D","@","W.T","*","df","*","g",")","@","B.T","dC\/dBg","=","A.T","@","X.T","@","(","D","@","W.T","*","df","*","g",")","Do","n't","forget","to","see","our","blog","post","for","more","details","!"]
code: def backward(ctx, dY: torch.Tensor):
      (
          gateW,
          gateW_quant,
          gateS,
          upW,
          upW_quant,
          upS,
          downW,
          downW_quant,
          downS,
      ) = ctx.custom_saved_tensors
      gateA, gateB, upA, upB, downA, downB, X, e, g = ctx.saved_tensors
  
      gateA, gateB, upA, upB, downA, downB = (
          gateA.t(),
          gateB.t(),
          upA.t(),
          upB.t(),
          downA.t(),
          downB.t(),
      )
  
      batch, seq_len, hd = X.shape
      dY = dY.view(-1, dY.shape[-1])
      X = X.view(-1, X.shape[-1])
      e = e.view(-1, e.shape[-1])
      g = g.view(-1, g.shape[-1])
      dtype = X.dtype
  
      # DW_f   = (D @ W.T * f)
      # DW_dfg = (D @ W.T * df * g)
      DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)
      DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g)
      h, DW_f, DW_dfg = DW, e, g
  
      # Down projection LoRA weights
      d_downA = h.t() @ (dY @ downB.t())
      d_downB = (downA.t() @ h.t()) @ dY
      d_downA *= downS
      d_downB *= downS
  
      # Up projection LoRA weights
      d_upA = X.t() @ (DW_f @ upB.t())
      d_upB = (upA.t() @ X.t()) @ DW_f
      d_upA *= upS
      d_upB *= upS
  
      # Gate projection LoRA weights
      d_gateA = X.t() @ (DW_dfg @ gateB.t())
      d_gateB = gateA.t() @ X.t() @ DW_dfg
      d_gateA *= gateS
      d_gateB *= gateS
  
      # Final derivatives to backpropagate backwards.
      # See our blogpost for more details.
      # (D @ W.T * f) @ U.T
      upW = fast_dequantize(upW.t(), upW_quant)
      # (D @ W.T * f) @ (U.T + B.T @ A.T)
      dX = torch.matmul(DW_f, upW.t(), out=X)
      del upW
      dX += DW_f @ upB.to(dtype).t() @ (upS * upA.to(dtype).t())
  
      # And add the derivative for the gate projection
      gateW = fast_dequantize(gateW.t(), gateW_quant)
      dX += DW_dfg @ gateW.t()
      del gateW
      dX += DW_dfg @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t())
  
      # gateW, gateW_quant, gateA, gateB, gateS,
      #  upW,    upW_quant,   upA,   upB,   upS,
      # downW, downW_quant, downA, downB, downS,
      return (
          dX.view(batch, seq_len, hd),
          None,
          None,
          d_gateA.t(),
          d_gateB.t(),
          None,
          None,
          None,
          d_upA.t(),
          d_upB.t(),
          None,
          None,
          None,
          d_downA.t(),
          d_downB.t(),
          None,
      )
  
file_path: unsloth/unsloth/kernels/fast_lora.py
imports: import torch
  from .utils import fast_dequantize, QUANT_STATE
  from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
label_desc: Use image node_id 2 for calling the LoRA_MLP obj's underlying member method code with example usage: obj.backward(ctx, dY) and returns: dX,  None,  None,  d_gateA,  d_gateB,  None,  None,  None,  d_upA,  d_upB,  None,  None,  None,  d_downA,  d_downB,  None