class_docstr: A wrapper of conditional attention, dropout and residual connection.
  
  Args:
      embed_dims (int): The embedding dimension.
      num_heads (int): Parallel attention heads.
      attn_drop (float): A Dropout layer on attn_output_weights.
          Default: 0.0.
      proj_drop: A Dropout layer after `nn.MultiheadAttention`.
          Default: 0.0.
      cross_attn (bool): Whether the attention module is for cross attention.
          Default: False
      keep_query_pos (bool): Whether to transform query_pos before cross
          attention.
          Default: False.
      batch_first (bool): When it is True, Key, Query and Value are shape of
          (batch, n, embed_dim), otherwise (n, batch, embed_dim).
           Default: True.
      init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization.
          Default: None.
class_docstr_tok: ["A","wrapper","of","conditional","attention",",","dropout","and","residual","connection",".","Args",":","embed_dims","(","int",")",":","The","embedding","dimension",".","num_heads","(","int",")",":","Parallel","attention","heads",".","attn_drop","(","float",")",":","A","Dropout","layer","on","attn_output_weights",".","Default",":","0.0.","proj_drop",":","A","Dropout","layer","after","`","nn.MultiheadAttention","`",".","Default",":","0.0.","cross_attn","(","bool",")",":","Whether","the","attention","module","is","for","cross","attention",".","Default",":","False","keep_query_pos","(","bool",")",":","Whether","to","transform","query_pos","before","cross","attention",".","Default",":","False",".","batch_first","(","bool",")",":","When","it","is","True",",","Key",",","Query","and","Value","are","shape","of","(","batch",",","n",",","embed_dim",")",",","otherwise","(","n",",","batch",",","embed_dim",")",".","Default",":","True",".","init_cfg","(","obj",":","`","mmcv.ConfigDict","`",")",":","The","Config","for","initialization",".","Default",":","None","."]
docstr: Forward process for `ConditionalAttention`.
  
  Args:
      query (Tensor): The input query with shape [bs, num_queries,
          embed_dims].
      key (Tensor): The key tensor with shape [bs, num_keys,
          embed_dims].
          If None, the `query` will be used. Defaults to None.
      value (Tensor): The value tensor with same shape as `key`.
          Same in `nn.MultiheadAttention.forward`. Defaults to None.
          If None, the `key` will be used.
      attn_mask (Tensor): ByteTensor mask with shape [num_queries,
          num_keys]. Same in `nn.MultiheadAttention.forward`.
          Defaults to None.
      key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys].
          Defaults to None.
  Returns:
      Tuple[Tensor]: Attention outputs of shape :math:`(N, L, E)`,
      where :math:`N` is the batch size, :math:`L` is the target
      sequence length , and :math:`E` is the embedding dimension
      `embed_dim`. Attention weights per head of shape :math:`
      (num_heads, L, S)`. where :math:`N` is batch size, :math:`L`
      is target sequence length, and :math:`S` is the source sequence
      length.
code: def forward_attn(
      self,
      query: Tensor,
      key: Tensor,
      value: Tensor,
      attn_mask: Tensor = None,
      key_padding_mask: Tensor = None,
  ) -> Tuple[Tensor]:
      """Forward process for `ConditionalAttention`.
  
      Args:
          query (Tensor): The input query with shape [bs, num_queries,
              embed_dims].
          key (Tensor): The key tensor with shape [bs, num_keys,
              embed_dims].
              If None, the `query` will be used. Defaults to None.
          value (Tensor): The value tensor with same shape as `key`.
              Same in `nn.MultiheadAttention.forward`. Defaults to None.
              If None, the `key` will be used.
          attn_mask (Tensor): ByteTensor mask with shape [num_queries,
              num_keys]. Same in `nn.MultiheadAttention.forward`.
              Defaults to None.
          key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys].
              Defaults to None.
      Returns:
          Tuple[Tensor]: Attention outputs of shape :math:`(N, L, E)`,
          where :math:`N` is the batch size, :math:`L` is the target
          sequence length , and :math:`E` is the embedding dimension
          `embed_dim`. Attention weights per head of shape :math:`
          (num_heads, L, S)`. where :math:`N` is batch size, :math:`L`
          is target sequence length, and :math:`S` is the source sequence
          length.
      """
      assert key.size(1) == value.size(
          1
      ), f'{"key, value must have the same sequence length"}'
      assert (
          query.size(0) == key.size(0) == value.size(0)
      ), f'{"batch size must be equal for query, key, value"}'
      assert query.size(2) == key.size(
          2
      ), f'{"q_dims, k_dims must be equal"}'
      assert (
          value.size(2) == self.embed_dims
      ), f'{"v_dims must be equal to embed_dims"}'
  
      bs, tgt_len, hidden_dims = query.size()
      _, src_len, _ = key.size()
      head_dims = hidden_dims // self.num_heads
      v_head_dims = self.embed_dims // self.num_heads
      assert (
          head_dims * self.num_heads == hidden_dims
      ), f'{"hidden_dims must be divisible by num_heads"}'
      scaling = float(head_dims) ** -0.5
  
      q = query * scaling
      k = key
      v = value
  
      if attn_mask is not None:
          assert (
              attn_mask.dtype == torch.float32
              or attn_mask.dtype == torch.float64
              or attn_mask.dtype == torch.float16
              or attn_mask.dtype == torch.uint8
              or attn_mask.dtype == torch.bool
          ), "Only float, byte, and bool types are supported for \
                      attn_mask"
  
          if attn_mask.dtype == torch.uint8:
              warnings.warn(
                  "Byte tensor for attn_mask is deprecated.\
                       Use bool tensor instead."
              )
              attn_mask = attn_mask.to(torch.bool)
          if attn_mask.dim() == 2:
              attn_mask = attn_mask.unsqueeze(0)
              if list(attn_mask.size()) != [
                  1,
                  query.size(1),
                  key.size(1),
              ]:
                  raise RuntimeError(
                      "The size of the 2D attn_mask is not correct."
                  )
          elif attn_mask.dim() == 3:
              if list(attn_mask.size()) != [
                  bs * self.num_heads,
                  query.size(1),
                  key.size(1),
              ]:
                  raise RuntimeError(
                      "The size of the 3D attn_mask is not correct."
                  )
          else:
              raise RuntimeError(
                  "attn_mask's dimension {} is not supported".format(
                      attn_mask.dim()
                  )
              )
      # attn_mask's dim is 3 now.
  
      if key_padding_mask is not None and key_padding_mask.dtype == int:
          key_padding_mask = key_padding_mask.to(torch.bool)
  
      q = (
          q.contiguous()
          .view(bs, tgt_len, self.num_heads, head_dims)
          .permute(0, 2, 1, 3)
          .flatten(0, 1)
      )
      if k is not None:
          k = (
              k.contiguous()
              .view(bs, src_len, self.num_heads, head_dims)
              .permute(0, 2, 1, 3)
              .flatten(0, 1)
          )
      if v is not None:
          v = (
              v.contiguous()
              .view(bs, src_len, self.num_heads, v_head_dims)
              .permute(0, 2, 1, 3)
              .flatten(0, 1)
          )
  
      if key_padding_mask is not None:
          assert key_padding_mask.size(0) == bs
          assert key_padding_mask.size(1) == src_len
  
      attn_output_weights = torch.bmm(q, k.transpose(1, 2))
      assert list(attn_output_weights.size()) == [
          bs * self.num_heads,
          tgt_len,
          src_len,
      ]
  
      if attn_mask is not None:
          if attn_mask.dtype == torch.bool:
              attn_output_weights.masked_fill_(attn_mask, float("-inf"))
          else:
              attn_output_weights += attn_mask
  
      if key_padding_mask is not None:
          attn_output_weights = attn_output_weights.view(
              bs, self.num_heads, tgt_len, src_len
          )
          attn_output_weights = attn_output_weights.masked_fill(
              key_padding_mask.unsqueeze(1).unsqueeze(2),
              float("-inf"),
          )
          attn_output_weights = attn_output_weights.view(
              bs * self.num_heads, tgt_len, src_len
          )
  
      attn_output_weights = F.softmax(
          attn_output_weights
          - attn_output_weights.max(dim=-1, keepdim=True)[0],
          dim=-1,
      )
      attn_output_weights = self.attn_drop(attn_output_weights)
  
      attn_output = torch.bmm(attn_output_weights, v)
      assert list(attn_output.size()) == [
          bs * self.num_heads,
          tgt_len,
          v_head_dims,
      ]
      attn_output = (
          attn_output.view(bs, self.num_heads, tgt_len, v_head_dims)
          .permute(0, 2, 1, 3)
          .flatten(2)
      )
      attn_output = self.out_proj(attn_output)
  
      # average attention weights over heads
      attn_output_weights = attn_output_weights.view(
          bs, self.num_heads, tgt_len, src_len
      )
      return (
          attn_output,
          attn_output_weights.sum(dim=1) / self.num_heads,
      )
  
imports: import math
  import warnings
  from typing import Optional, Sequence, Tuple, Union
  import torch
  import torch.nn.functional
  from mmcv.cnn import Linear, build_activation_layer, build_conv_layer, build_norm_layer
  from mmcv.cnn.bricks.drop import Dropout
  from mmengine.model import BaseModule, ModuleList
  from mmengine.utils import to_2tuple
  from torch import Tensor, nn
  from mmdet.registry import MODELS
  from mmdet.utils import OptConfigType, OptMultiConfig
label_desc: Use image node_id 3 for calling the ConditionalAttention obj's underlying member method code with example usage: obj.forward_attn(query, key, value, attn_mask, key_padding_mask) and returns: attn_output,  unknown