def _split_heads(self, tensor, num_heads, attn_head_size):
    '\n        Splits hidden_size dim into attn_head_size and num_heads\n        '
    new_shape = (tensor.size()[:(- 1)] + (num_heads, attn_head_size))
    tensor = tensor.view(*new_shape)
    if (len(tensor.shape) == 5):
        return tensor.permute(0, 1, 3, 2, 4)
    elif (len(tensor.shape) == 4):
        return tensor.permute(0, 2, 1, 3)
    else:
        raise ValueError(f'Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}')
Splits hidden_size dim into attn_head_size and num_heads
def _split_heads(self, tensor, num_heads, attn_head_size):
    '\n        \n        '
    new_shape = (tensor.size()[:(- 1)] + (num_heads, attn_head_size))
    tensor = tensor.view(*new_shape)
    if (len(tensor.shape) == 5):
        return tensor.permute(0, 1, 3, 2, 4)
    elif (len(tensor.shape) == 4):
        return tensor.permute(0, 2, 1, 3)
    else:
        raise ValueError(f'Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}')