class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
class_docstr_tok: ["Multi-headed","attention","from","'Attention","Is","All","You","Need","'","paper"]
code: def __init__(
      self,
      embed_dim: int,
      num_heads: int,
      head_dim: int,
      dropout: float = 0.0,
      is_decoder: bool = False,
      bias: bool = False,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.num_heads = num_heads
      self.dropout = dropout
      self.head_dim = head_dim
      self.inner_dim = head_dim * num_heads
  
      self.scaling = self.head_dim**-0.5
      self.is_decoder = is_decoder
  
      self.k_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias)
      self.v_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias)
      self.q_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias)
      self.out_proj = nn.Linear(self.inner_dim, embed_dim)
  
imports: import inspect
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn
  import torch.utils.checkpoint
  from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer
  from transformers.activations import ACT2FN
  from transformers.modeling_outputs import BaseModelOutput
  from transformers.utils import logging
  from ...models import AutoencoderKL, UNet2DConditionModel, UNet2DModel, VQModel
  from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler
  from ...utils.torch_utils import randn_tensor
  from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput
label_desc: Use image node_id 1 to create a new LDMBertAttention object from inherited base classes: nn with example: obj = LDMBertAttention(embed_dim, num_heads, head_dim, dropout, is_decoder, bias)