paddlenlp/transformers/distill_utils.py
def calc_multi_relation_loss(loss_fct,
                             s,
                             t,
                             attn_mask,
                             num_relation_heads=0,
                             alpha=0.0,
                             beta=0.0):
    """
    Calculates loss for multiple Q-Q, K-K and V-V relation. It supports
    head-head relation, sample-sample relation and origin token-token relation.
    The final loss value could be balanced by weight `alpha` and `beta`.

    Args:
        loss_fct (callable):
            Loss function for distillation. It only supports kl_div loss now.
        s (Tensor):
            Q, K, V of Student.
        t (Tensor):
            Q, K, V of teacher.
        attn_mask (Tensor):
            Attention mask for relation.
        num_relation_heads (int):
            The number of relation heads. 0 means `num_relation_heads` equals
            to origin head num.
            Defaults to 0.
        alpha (float):
            The weight for head-head relation.
            Defaults to 0.0.
        beta (float):
            The weight for sample-sample relation.
            Defaults to 0.0.

    Returns:
        Tensor: Weighted loss of token-token loss, head-head loss and
            sample-sample loss.

    """
    # Initialize head_num
    if num_relation_heads > 0 and num_relation_heads != s.shape[1]:
        # s'shape: [bs, seq_len, head_num, head_dim]
        s = tensor.transpose(x=s, perm=[0, 2, 1, 3])
        # s'shape: [bs, seq_len, num_relation_heads, head_dim_new]
        s = tensor.reshape(x=s, shape=[0, 0, num_relation_heads, -1])
        s1 = tensor.transpose(x=s, perm=[0, 2, 1, 3])
    if num_relation_heads > 0 and num_relation_heads != t.shape[1]:
        t = tensor.transpose(x=t, perm=[0, 2, 1, 3])
        t = tensor.reshape(x=t, shape=[0, 0, num_relation_heads, -1])
        t1 = tensor.transpose(x=t, perm=[0, 2, 1, 3])

    s_head_dim, t_head_dim = s.shape[3], t.shape[3]

    if alpha + beta == 1.0:
        loss_token_token = 0.0
    else:
        scaled_dot_product_s1 = tensor.matmul(
            x=s1, y=s1, transpose_y=True) / math.sqrt(s_head_dim)
        del s1
        scaled_dot_product_s1 += attn_mask
        scaled_dot_product_t1 = tensor.matmul(
            x=t1, y=t1, transpose_y=True) / math.sqrt(t_head_dim)
        del t1
        scaled_dot_product_t1 += attn_mask
        loss_token_token = loss_fct(
            F.log_softmax(scaled_dot_product_s1),
            F.softmax(scaled_dot_product_t1))

    if alpha == 0.0:
        loss_head_head = 0.0
    else:
        scaled_dot_product_s = tensor.matmul(
            x=s, y=s, transpose_y=True) / math.sqrt(s_head_dim)
        attn_mask_head_head = tensor.transpose(x=attn_mask, perm=[0, 3, 1, 2])

        scaled_dot_product_s += attn_mask_head_head
        scaled_dot_product_t = tensor.matmul(
            x=t, y=t, transpose_y=True) / math.sqrt(t_head_dim)
        scaled_dot_product_t += attn_mask_head_head
        loss_head_head = loss_fct(
            F.log_softmax(scaled_dot_product_s),
            F.softmax(scaled_dot_product_t))
    if beta == 0.0:
        loss_sample_sample = 0.0
    else:
        s2 = tensor.transpose(x=s, perm=[1, 2, 0, 3])
        scaled_dot_product_s2 = tensor.matmul(
            x=s2, y=s2, transpose_y=True) / math.sqrt(s_head_dim)

        del s, s2
        # Shape: [seq_len, 1, batch_size, 1]
        attn_mask_sample_sample = tensor.transpose(
            x=attn_mask, perm=[3, 1, 0, 2])

        # Shape: [seq_len, head_num, batch_size, batch_size]
        scaled_dot_product_s2 += attn_mask_sample_sample
        t2 = tensor.transpose(x=t, perm=[1, 2, 0, 3])
        scaled_dot_product_t2 = tensor.matmul(
            x=t2, y=t2, transpose_y=True) / math.sqrt(t_head_dim)

        del t, t2
        scaled_dot_product_t2 += attn_mask_sample_sample
        loss_sample_sample = loss_fct(
            F.log_softmax(scaled_dot_product_s2),
            F.softmax(scaled_dot_product_t2))

    return (
        1 - alpha - beta
    ) * loss_token_token + alpha * loss_head_head + beta * loss_sample_sample

    Calculates loss for multiple Q-Q, K-K and V-V relation. It supports
    head-head relation, sample-sample relation and origin token-token relation.
    The final loss value could be balanced by weight `alpha` and `beta`.

    Args:
        loss_fct (callable):
            Loss function for distillation. It only supports kl_div loss now.
        s (Tensor):
            Q, K, V of Student.
        t (Tensor):
            Q, K, V of teacher.
        attn_mask (Tensor):
            Attention mask for relation.
        num_relation_heads (int):
            The number of relation heads. 0 means `num_relation_heads` equals
            to origin head num.
            Defaults to 0.
        alpha (float):
            The weight for head-head relation.
            Defaults to 0.0.
        beta (float):
            The weight for sample-sample relation.
            Defaults to 0.0.

    Returns:
        Tensor: Weighted loss of token-token loss, head-head loss and
            sample-sample loss.

    
Calculates loss for multiple Q-Q, K-K and V-V relation. It supports
head-head relation, sample-sample relation and origin token-token relation.
The final loss value could be balanced by weight `alpha` and `beta`.
Calculates loss for multiple Q - Q K - K and V - V relation . It supports head - head relation sample - sample relation and origin token - token relation . The final loss value could be balanced by weight ` alpha ` and ` beta ` .
def calc_multi_relation_loss(loss_fct,
                             s,
                             t,
                             attn_mask,
                             num_relation_heads=0,
                             alpha=0.0,
                             beta=0.0):
    if num_relation_heads > 0 and num_relation_heads != s.shape[1]:
        s = tensor.transpose(x=s, perm=[0, 2, 1, 3])
        s = tensor.reshape(x=s, shape=[0, 0, num_relation_heads, -1])
        s1 = tensor.transpose(x=s, perm=[0, 2, 1, 3])
    if num_relation_heads > 0 and num_relation_heads != t.shape[1]:
        t = tensor.transpose(x=t, perm=[0, 2, 1, 3])
        t = tensor.reshape(x=t, shape=[0, 0, num_relation_heads, -1])
        t1 = tensor.transpose(x=t, perm=[0, 2, 1, 3])
    s_head_dim, t_head_dim = s.shape[3], t.shape[3]
    if alpha + beta == 1.0:
        loss_token_token = 0.0
    else:
        scaled_dot_product_s1 = tensor.matmul(
            x=s1, y=s1, transpose_y=True) / math.sqrt(s_head_dim)
        del s1
        scaled_dot_product_s1 += attn_mask
        scaled_dot_product_t1 = tensor.matmul(
            x=t1, y=t1, transpose_y=True) / math.sqrt(t_head_dim)
        del t1
        scaled_dot_product_t1 += attn_mask
        loss_token_token = loss_fct(
            F.log_softmax(scaled_dot_product_s1),
            F.softmax(scaled_dot_product_t1))
    if alpha == 0.0:
        loss_head_head = 0.0
    else:
        scaled_dot_product_s = tensor.matmul(
            x=s, y=s, transpose_y=True) / math.sqrt(s_head_dim)
        attn_mask_head_head = tensor.transpose(x=attn_mask, perm=[0, 3, 1, 2])
        scaled_dot_product_s += attn_mask_head_head
        scaled_dot_product_t = tensor.matmul(
            x=t, y=t, transpose_y=True) / math.sqrt(t_head_dim)
        scaled_dot_product_t += attn_mask_head_head
        loss_head_head = loss_fct(
            F.log_softmax(scaled_dot_product_s),
            F.softmax(scaled_dot_product_t))
    if beta == 0.0:
        loss_sample_sample = 0.0
    else:
        s2 = tensor.transpose(x=s, perm=[1, 2, 0, 3])
        scaled_dot_product_s2 = tensor.matmul(
            x=s2, y=s2, transpose_y=True) / math.sqrt(s_head_dim)
        del s, s2
        attn_mask_sample_sample = tensor.transpose(
            x=attn_mask, perm=[3, 1, 0, 2])
        scaled_dot_product_s2 += attn_mask_sample_sample
        t2 = tensor.transpose(x=t, perm=[1, 2, 0, 3])
        scaled_dot_product_t2 = tensor.matmul(
            x=t2, y=t2, transpose_y=True) / math.sqrt(t_head_dim)
        del t, t2
        scaled_dot_product_t2 += attn_mask_sample_sample
        loss_sample_sample = loss_fct(
            F.log_softmax(scaled_dot_product_s2),
            F.softmax(scaled_dot_product_t2))
    return (
        1 - alpha - beta
    ) * loss_token_token + alpha * loss_head_head + beta * loss_sample_sample
def calc_multi_relation_loss ( loss_fct , s , t , attn_mask , num_relation_heads = 0 , alpha = 0.0 , beta = 0.0 ) : if num_relation_heads > 0 and num_relation_heads != s . shape [ 1 ] : s = tensor . transpose ( x = s , perm = [ 0 , 2 , 1 , 3 ] ) s = tensor . reshape ( x = s , shape = [ 0 , 0 , num_relation_heads , - 1 ] ) s1 = tensor . transpose ( x = s , perm = [ 0 , 2 , 1 , 3 ] ) if num_relation_heads > 0 and num_relation_heads != t . shape [ 1 ] : t = tensor . transpose ( x = t , perm = [ 0 , 2 , 1 , 3 ] ) t = tensor . reshape ( x = t , shape = [ 0 , 0 , num_relation_heads , - 1 ] ) t1 = tensor . transpose ( x = t , perm = [ 0 , 2 , 1 , 3 ] ) s_head_dim , t_head_dim = s . shape [ 3 ] , t . shape [ 3 ] if alpha + beta == 1.0 : loss_token_token = 0.0 else : scaled_dot_product_s1 = tensor . matmul ( x = s1 , y = s1 , transpose_y = True ) / math . sqrt ( s_head_dim ) del s1 scaled_dot_product_s1 += attn_mask scaled_dot_product_t1 = tensor . matmul ( x = t1 , y = t1 , transpose_y = True ) / math . sqrt ( t_head_dim ) del t1 scaled_dot_product_t1 += attn_mask loss_token_token = loss_fct ( F . log_softmax ( scaled_dot_product_s1 ) , F . softmax ( scaled_dot_product_t1 ) ) if alpha == 0.0 : loss_head_head = 0.0 else : scaled_dot_product_s = tensor . matmul ( x = s , y = s , transpose_y = True ) / math . sqrt ( s_head_dim ) attn_mask_head_head = tensor . transpose ( x = attn_mask , perm = [ 0 , 3 , 1 , 2 ] ) scaled_dot_product_s += attn_mask_head_head scaled_dot_product_t = tensor . matmul ( x = t , y = t , transpose_y = True ) / math . sqrt ( t_head_dim ) scaled_dot_product_t += attn_mask_head_head loss_head_head = loss_fct ( F . log_softmax ( scaled_dot_product_s ) , F . softmax ( scaled_dot_product_t ) ) if beta == 0.0 : loss_sample_sample = 0.0 else : s2 = tensor . transpose ( x = s , perm = [ 1 , 2 , 0 , 3 ] ) scaled_dot_product_s2 = tensor . matmul ( x = s2 , y = s2 , transpose_y = True ) / math . sqrt ( s_head_dim ) del s , s2 attn_mask_sample_sample = tensor . transpose ( x = attn_mask , perm = [ 3 , 1 , 0 , 2 ] ) scaled_dot_product_s2 += attn_mask_sample_sample t2 = tensor . transpose ( x = t , perm = [ 1 , 2 , 0 , 3 ] ) scaled_dot_product_t2 = tensor . matmul ( x = t2 , y = t2 , transpose_y = True ) / math . sqrt ( t_head_dim ) del t , t2 scaled_dot_product_t2 += attn_mask_sample_sample loss_sample_sample = loss_fct ( F . log_softmax ( scaled_dot_product_s2 ) , F . softmax ( scaled_dot_product_t2 ) ) return ( 1 - alpha - beta ) * loss_token_token + alpha * loss_head_head + beta * loss_sample_sample
Calculates loss for multiple Q-Q, K-K and V-V relation.
Calculates loss for multiple Q - Q K - K and V - V relation .
"""
    Calculates loss for multiple Q-Q, K-K and V-V relation. It supports
    head-head relation, sample-sample relation and origin token-token relation.
    The final loss value could be balanced by weight `alpha` and `beta`.

    Args:
        loss_fct (callable):
            Loss function for distillation. It only supports kl_div loss now.
        s (Tensor):
            Q, K, V of Student.
        t (Tensor):
            Q, K, V of teacher.
        attn_mask (Tensor):
            Attention mask for relation.
        num_relation_heads (int):
            The number of relation heads. 0 means `num_relation_heads` equals
            to origin head num.
            Defaults to 0.
        alpha (float):
            The weight for head-head relation.
            Defaults to 0.0.
        beta (float):
            The weight for sample-sample relation.
            Defaults to 0.0.

    Returns:
        Tensor: Weighted loss of token-token loss, head-head loss and
            sample-sample loss.

    """ # Initialize head_num # s'shape: [bs, seq_len, head_num, head_dim] # s'shape: [bs, seq_len, num_relation_heads, head_dim_new] # Shape: [seq_len, 1, batch_size, 1] # Shape: [seq_len, head_num, batch_size, batch_size]
None
Tensor
loss_fct
Loss function for distillation. It only supports kl_div loss now.
Loss function for distillation . It only supports kl_div loss now .
s
Q, K, V of Student.
Q K V of Student .
t
Q, K, V of teacher.
Q K V of teacher .
attn_mask
Attention mask for relation.
Attention mask for relation .
num_relation_heads
The number of relation heads. 0 means `num_relation_heads` equals
to origin head num.
Defaults to 0.
The number of relation heads . 0 means ` num_relation_heads ` equals to origin head num . Defaults to 0 .
alpha
The weight for head-head relation.
Defaults to 0.0.
The weight for head - head relation . Defaults to 0 . 0 .
beta
The weight for sample-sample relation.
Defaults to 0.0.
The weight for sample - sample relation . Defaults to 0 . 0 .