class_docstr: Cross-Covariance Attention (XCA) operation where the channels are updated using
  a weighted sum. The weights are obtained from the (softmax normalized)
  Cross-covariance matrix (Q^T K \in d_h \times d_h)
class_docstr_tok: ["Cross-Covariance","Attention","(","XCA",")","operation","where","the","channels","are","updated","using","a","weighted","sum",".","The","weights","are","obtained","from","the","(","softmax","normalized",")","Cross-covariance","matrix","(","Q^T","K","\\in","d_h","\\times","d_h",")"]
code: def __init__(
      self,
      dim,
      num_heads=8,
      qkv_bias=False,
      qk_scale=None,
      attn_drop=0.0,
      proj_drop=0.0,
  ):
      super().__init__()
      self.num_heads = num_heads
      self.temperature = nn.Parameter(torch.ones(num_heads, 1, 1))
  
      self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
      self.attn_drop = nn.Dropout(attn_drop)
      self.proj = nn.Linear(dim, dim)
      self.proj_drop = nn.Dropout(proj_drop)
  
imports: import logging
  import math
  from functools import partial
  from typing import List, Optional
  import torch
  import torch.nn
  from fairscale.nn import checkpoint_wrapper
  from vissl.config import AttrDict
  from vissl.models.model_helpers import DropPath, to_2tuple, trunc_normal_
  from vissl.models.trunks import register_model_trunk
  from vissl.models.trunks.vision_transformer import Mlp
  from vissl.utils.fsdp_utils import fsdp_wrapper
  from vissl.utils.misc import set_torch_seed
label_desc: Use image node_id 1 to create a new XCA object from inherited base classes: nn with example: obj = XCA(dim, num_heads, qkv_bias, qk_scale, attn_drop, proj_drop)