class_docstr: ### LoRA weights
  Wq = Wq + Aq @ Bq
  Wk = Wk + Ak @ Bk
  Wv = Wv + Av @ Bv
  Q = X @ Wq = X @ Wq + X @ Aq @ Bq
  K = X @ Wk = X @ Wk + X @ Ak @ Bk
  V = X @ Wv = X @ Wv + X @ Av @ Bv
  
  ### Backpropagation chain rule
  See our blogpost for more details.
  
  dC/dWq = X.T @ D(Wq)
  dC/dWk = X.T @ D(Wk)
  dC/dWv = X.T @ D(Wv)
  We then sum them all find dC/dX
  
  ### Q projection LoRA weights
  dC/dAq =       X.T @ D(Wq) @ B.T
  dC/dBq = A.T @ X.T @ D(Wq)
  
  ### K projection LoRA weights
  dC/dAk =       X.T @ D(Wk) @ B.T
  dC/dBk = A.T @ X.T @ D(Wk)
  
  ### V projection LoRA weights
  dC/dAv =       X.T @ D(Wv) @ B.T
  dC/dBv = A.T @ X.T @ D(Wv)
class_docstr_tok: ["#","#","#","LoRA","weights","Wq","=","Wq","+","Aq","@","Bq","Wk","=","Wk","+","Ak","@","Bk","Wv","=","Wv","+","Av","@","Bv","Q","=","X","@","Wq","=","X","@","Wq","+","X","@","Aq","@","Bq","K","=","X","@","Wk","=","X","@","Wk","+","X","@","Ak","@","Bk","V","=","X","@","Wv","=","X","@","Wv","+","X","@","Av","@","Bv","#","#","#","Backpropagation","chain","rule","See","our","blogpost","for","more","details",".","dC\/dWq","=","X.T","@","D","(","Wq",")","dC\/dWk","=","X.T","@","D","(","Wk",")","dC\/dWv","=","X.T","@","D","(","Wv",")","We","then","sum","them","all","find","dC\/dX","#","#","#","Q","projection","LoRA","weights","dC\/dAq","=","X.T","@","D","(","Wq",")","@","B.T","dC\/dBq","=","A.T","@","X.T","@","D","(","Wq",")","#","#","#","K","projection","LoRA","weights","dC\/dAk","=","X.T","@","D","(","Wk",")","@","B.T","dC\/dBk","=","A.T","@","X.T","@","D","(","Wk",")","#","#","#","V","projection","LoRA","weights","dC\/dAv","=","X.T","@","D","(","Wv",")","@","B.T","dC\/dBv","=","A.T","@","X.T","@","D","(","Wv",")"]
code: def backward(ctx, dQ, dK, dV):
      (
          QW,
          QW_quant,
          QS,
          KW,
          KW_quant,
          KS,
          VW,
          VW_quant,
          VS,
      ) = ctx.custom_saved_tensors
      (
          X,
          QA,
          QB,
          KA,
          KB,
          VA,
          VB,
      ) = ctx.saved_tensors
  
      QA, QB, KA, KB, VA, VB = (
          QA.t(),
          QB.t(),
          KA.t(),
          KB.t(),
          VA.t(),
          VB.t(),
      )
  
      batch, seq_len, hd = X.shape
      dQ = dQ.view(-1, dQ.shape[-1])
      dK = dK.reshape(-1, dK.shape[-1])  # view doesn't work on K.T
      dV = dV.view(-1, dV.shape[-1])
      X = X.view(-1, X.shape[-1])
      dtype = X.dtype
  
      ### Weight projection LoRA weights
      # See our blogpost for more details.
  
      # Q Projection
      d_QA = X.t() @ (dQ @ QB.t())
      d_QB = (QA.t() @ X.t()) @ dQ
      d_QA *= QS
      d_QB *= QS
  
      # K Projection
      d_KA = X.t() @ (dK @ KB.t())
      d_KB = (KA.t() @ X.t()) @ dK
      d_KA *= KS
      d_KB *= KS
  
      # V Projection
      d_VA = X.t() @ (dV @ VB.t())
      d_VB = (VA.t() @ X.t()) @ dV
      d_VA *= VS
      d_VB *= VS
  
      # Combine derivatives to find dX
      # dQ
      QW = fast_dequantize(QW.t(), QW_quant)
      dX = torch.matmul(dQ, QW.t(), out=X)
      del QW
      dX += dQ @ QB.to(dtype).t() @ (QS * QA.to(dtype).t())
  
      # dK
      KW = fast_dequantize(KW.t(), KW_quant)
      dX += dK @ KW.t()
      del KW
      dX += dK @ KB.to(dtype).t() @ (KS * KA.to(dtype).t())
  
      # dV
      VW = fast_dequantize(VW.t(), VW_quant)
      dX += dV @ VW.t()
      del VW
      dX += dV @ VB.to(dtype).t() @ (VS * VA.to(dtype).t())
  
      # QW, QW_quant, QA, QB, QS,
      # KW, KW_quant, KA, KB, KS,
      # VW, VW_quant, VA, VB, VS,
      return (
          dX.view(batch, seq_len, hd),
          None,
          None,
          d_QA.t(),
          d_QB.t(),
          None,
          None,
          None,
          d_KA.t(),
          d_KB.t(),
          None,
          None,
          None,
          d_VA.t(),
          d_VB.t(),
          None,
      )
  
file_path: unsloth/unsloth/kernels/fast_lora.py
imports: import torch
  from .utils import fast_dequantize, QUANT_STATE
  from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
label_desc: Use image node_id 2 for calling the LoRA_QKV obj's underlying member method code with example usage: obj.backward(ctx, dQ, dK, dV) and returns: dX,  None,  None,  d_QA,  d_QB,  None,  None,  None,  d_KA,  d_KB,  None,  None,  None,  d_VA,  d_VB,  None