Fixed DeBERTa v2 attention calculation
@@ -251,7 +251,9 @@ impl DisentangledSelfAttention {
let mut score = Tensor::zeros(&[1], (query_layer.kind(), query_layer.device()));
- let c2p_pos = if self.pos_att_type.has_type(PositionAttentionType::c2p) {
+ let c2p_pos = if self.pos_att_type.has_type(PositionAttentionType::c2p)
+ | self.pos_att_type.has_type(PositionAttentionType::p2p)
+ {
let scale = *pos_key_layer.size().last().unwrap() as f64 * scale_factor;
let c2p_att = query_layer.bmm(&pos_key_layer.transpose(-1, -2));
let c2p_pos = relative_pos.clamp(0, att_span * 2 - 1);
@@ -273,9 +275,7 @@ impl DisentangledSelfAttention {
None
};
- if self.pos_att_type.has_type(PositionAttentionType::p2c)
- | self.pos_att_type.has_type(PositionAttentionType::p2p)
- {
+ if self.pos_att_type.has_type(PositionAttentionType::p2c) {
let scale = *pos_query_layer.size().last().unwrap() as f64 * scale_factor;
let r_pos = if key_layer_size[1] != query_layer_size[1] {
build_relative_position(
@@ -292,7 +292,6 @@ impl DisentangledSelfAttention {
let p2c_pos = (-r_pos + att_span).clamp(0, 2 * att_span - 1);
- if self.pos_att_type.has_type(PositionAttentionType::p2c) {
let p2c_att = key_layer
.bmm(&pos_query_layer.transpose(-1, -2))
.gather(
@@ -327,7 +326,6 @@ impl DisentangledSelfAttention {
);
score = score + p2p_att / scale;
}
- }
Ok(Tensor::new())
}
