class_docstr: ### LoRA weights
  Wq = Wq + Aq @ Bq
  Wk = Wk + Ak @ Bk
  Wv = Wv + Av @ Bv
  Q = X @ Wq = X @ Wq + X @ Aq @ Bq
  K = X @ Wk = X @ Wk + X @ Ak @ Bk
  V = X @ Wv = X @ Wv + X @ Av @ Bv
  
  ### Backpropagation chain rule
  dC/dWq = X.T @ D(Wq)
  dC/dWk = X.T @ D(Wk)
  dC/dWv = X.T @ D(Wv)
  
  ### Q projection LoRA weights
  dC/dAq =       X.T @ D(Wq) @ B.T
  dC/dBq = A.T @ X.T @ D(Wq)
  
  ### K projection LoRA weights
  dC/dAk =       X.T @ D(Wk) @ B.T
  dC/dBk = A.T @ X.T @ D(Wk)
  
  ### V projection LoRA weights
  dC/dAv =       X.T @ D(Wv) @ B.T
  dC/dBv = A.T @ X.T @ D(Wv)
class_docstr_tok: ["#","#","#","LoRA","weights","Wq","=","Wq","+","Aq","@","Bq","Wk","=","Wk","+","Ak","@","Bk","Wv","=","Wv","+","Av","@","Bv","Q","=","X","@","Wq","=","X","@","Wq","+","X","@","Aq","@","Bq","K","=","X","@","Wk","=","X","@","Wk","+","X","@","Ak","@","Bk","V","=","X","@","Wv","=","X","@","Wv","+","X","@","Av","@","Bv","#","#","#","Backpropagation","chain","rule","dC\/dWq","=","X.T","@","D","(","Wq",")","dC\/dWk","=","X.T","@","D","(","Wk",")","dC\/dWv","=","X.T","@","D","(","Wv",")","#","#","#","Q","projection","LoRA","weights","dC\/dAq","=","X.T","@","D","(","Wq",")","@","B.T","dC\/dBq","=","A.T","@","X.T","@","D","(","Wq",")","#","#","#","K","projection","LoRA","weights","dC\/dAk","=","X.T","@","D","(","Wk",")","@","B.T","dC\/dBk","=","A.T","@","X.T","@","D","(","Wk",")","#","#","#","V","projection","LoRA","weights","dC\/dAv","=","X.T","@","D","(","Wv",")","@","B.T","dC\/dBv","=","A.T","@","X.T","@","D","(","Wv",")"]
code: def backward(ctx, dY: torch.Tensor):
      W, W_quant, S = ctx.custom_saved_tensors
      A, B, X = ctx.saved_tensors
  
      A, B = A.t(), B.t()
  
      batch, seq_len, hd = X.shape
      dY = dY.reshape(-1, dY.shape[-1])  # Must be reshape
      X = X.reshape(-1, X.shape[-1])  # Must be reshape
      dtype = X.dtype
  
      ### Weight projection LoRA weights
      # Weight projection
      d_A = X.t() @ (dY @ B.t())
      d_B = (A.t() @ X.t()) @ dY
      d_A *= S
      d_B *= S
  
      # Get derivative for dX
      W = fast_dequantize(W.t(), W_quant)
      dX = dY @ W.t()
      del W
      dX += dY @ B.to(dtype).t() @ (S * A.to(dtype).t())
  
      # W, W_quant, A, B, S
      return (
          dX.view(batch, seq_len, hd),
          None,
          None,
          d_A.t(),
          d_B.t(),
          None,
      )
  
file_path: unsloth/unsloth/kernels/fast_lora.py
imports: import torch
  from .utils import fast_dequantize, QUANT_STATE
  from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
label_desc: Use image node_id 2 for calling the LoRA_W obj's underlying member method code with example usage: obj.backward(ctx, dY) and returns: dX,  None,  None,  d_A,  d_B,  None