class_docstr: Cross-Covariance Attention (XCA) operation where the channels are updated using
  a weighted sum. The weights are obtained from the (softmax normalized)
  Cross-covariance matrix (Q^T K \in d_h \times d_h)
class_docstr_tok: ["Cross-Covariance","Attention","(","XCA",")","operation","where","the","channels","are","updated","using","a","weighted","sum",".","The","weights","are","obtained","from","the","(","softmax","normalized",")","Cross-covariance","matrix","(","Q^T","K","\\in","d_h","\\times","d_h",")"]
code: def forward(self, x):
      B, N, C = x.shape
      qkv = self.qkv(x).reshape(
          B, N, 3, self.num_heads, C // self.num_heads
      )
      qkv = qkv.permute(2, 0, 3, 1, 4)
      q, k, v = (
          qkv[0],
          qkv[1],
          qkv[2],
      )  # make torchscript happy (cannot use tensor as tuple)
  
      q = q.transpose(-2, -1)
      k = k.transpose(-2, -1)
      v = v.transpose(-2, -1)
  
      q = torch.nn.functional.normalize(q, dim=-1)
      k = torch.nn.functional.normalize(k, dim=-1)
  
      attn = (q @ k.transpose(-2, -1)) * self.temperature
      attn = attn.softmax(dim=-1)
      attn = self.attn_drop(attn)
  
      x = (attn @ v).permute(0, 3, 1, 2).reshape(B, N, C)
      x = self.proj(x)
      x = self.proj_drop(x)
      return x
  
imports: import logging
  import math
  from functools import partial
  from typing import List, Optional
  import torch
  import torch.nn
  from fairscale.nn import checkpoint_wrapper
  from vissl.config import AttrDict
  from vissl.models.model_helpers import DropPath, to_2tuple, trunc_normal_
  from vissl.models.trunks import register_model_trunk
  from vissl.models.trunks.vision_transformer import Mlp
  from vissl.utils.fsdp_utils import fsdp_wrapper
  from vissl.utils.misc import set_torch_seed
label_desc: Use image node_id 2 for calling the XCA obj's underlying member method code with example usage: obj.forward(x) and returns: x