def forward(self, qkv):
    '\n        Apply QKV attention.\n\n        :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs.\n        :return: an [N x (H * C) x T] tensor after attention.\n        '
    (bs, width, length) = qkv.shape
    assert ((width % (3 * self.n_heads)) == 0)
    ch = (width // (3 * self.n_heads))
    (q, k, v) = qkv.chunk(3, dim=1)
    scale = (1 / math.sqrt(math.sqrt(ch)))
    weight = th.einsum('bct,bcs->bts', (q * scale).view((bs * self.n_heads), ch, length), (k * scale).view((bs * self.n_heads), ch, length))
    weight = th.softmax(weight.float(), dim=(- 1)).type(weight.dtype)
    a = th.einsum('bts,bcs->bct', weight, v.reshape((bs * self.n_heads), ch, length))
    return a.reshape(bs, (- 1), length)
Apply QKV attention.

:param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs.
:return: an [N x (H * C) x T] tensor after attention.
def forward(self, qkv):
    '\n        Apply QKV attention.\n\n        :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs.\n        :return: an [N x (H * C) x T] tensor after attention.\n        '
    (bs, width, length) = qkv.shape
    assert ((width % (3 * self.n_heads)) == 0)
    ch = (width // (3 * self.n_heads))
    (q, k, v) = qkv.chunk(3, dim=1)
    scale = (1 / math.sqrt(math.sqrt(ch)))
    weight = th.einsum('bct,bcs->bts', (q * scale).view((bs * self.n_heads), ch, length), (k * scale).view((bs * self.n_heads), ch, length))
    weight = th.softmax(weight.float(), dim=(- 1)).type(weight.dtype)
    a = th.einsum('bts,bcs->bct', weight, v.reshape((bs * self.n_heads), ch, length))
    return a.reshape(bs, (- 1), length)