class_docstr: This is DP-friendly implementation of nn.MultiheadAttention.
  For full reference see original module refer to
  :class:`torch.nn.MultiheadAttention`.
  
  Current implementation leverages pytorch modules as building blocks
  to allow DP engine to calculate per-sample gradients.
  This is in contrast with original implementation based on nn.functional.
class_docstr_tok: ["This","is","DP-friendly","implementation","of","nn.MultiheadAttention",".","For","full","reference","see","original","module","refer","to",":","class",":","`","torch.nn.MultiheadAttention","`",".","Current","implementation","leverages","pytorch","modules","as","building","blocks","to","allow","DP","engine","to","calculate","per-sample","gradients",".","This","is","in","contrast","with","original","implementation","based","on","nn.functional","."]
code: def __init__(
      self,
      embed_dim,
      num_heads,
      dropout=0.0,
      bias=True,
      add_bias_kv=False,
      add_zero_attn=False,
      kdim=None,
      vdim=None,
      batch_first=False,
      device=None,
      dtype=None,
  ):
      super(DPMultiheadAttention, self).__init__()
      self.embed_dim = embed_dim
      self.kdim = kdim if kdim is not None else embed_dim
      self.vdim = vdim if vdim is not None else embed_dim
  
      # when self._qkv_same_embed_dim = True, "in_proj_weight" rather than "q,k,v_weight" and fast path calculation will be used in "nn.transformer", which should be avoided. This is why we force self._qkv_same_embed_dim = False.
      self._qkv_same_embed_dim = False
  
      self.num_heads = num_heads
      self.dropout = dropout
      self.batch_first = batch_first
      self.head_dim = embed_dim // num_heads
      assert (
          self.head_dim * num_heads == self.embed_dim
      ), "embed_dim must be divisible by num_heads"
  
      self.qlinear = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.klinear = nn.Linear(self.kdim, embed_dim, bias=bias)
      self.vlinear = nn.Linear(self.vdim, embed_dim, bias=bias)
  
      self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
  
      self.add_bias_kv = add_bias_kv
      if self.add_bias_kv:
          self.seq_bias_k = SequenceBias(embed_dim)
          self.seq_bias_v = SequenceBias(embed_dim)
  
      self.add_zero_attn = add_zero_attn
  
      self.dropout = nn.Dropout(dropout)
  
      # to avoid null pointers in Transformer.forward
      self.in_proj_weight = None
      self.in_proj_bias = None
  
file_path: opacus/opacus/layers/dp_multihead_attention.py
imports: import warnings
  from collections import OrderedDict
  import torch
  import torch.nn
  import torch.nn.functional
  from torch.nn.parameter import Parameter
label_desc: Use image node_id 1 to create a new DPMultiheadAttention object from inherited base classes: nn with example: obj = DPMultiheadAttention(embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype)