util/loss_contrastive.py

# -*- coding: utf-8 -*-
# Author: Gaojian Wang@ZJUICSR
# --------------------------------------------------------
# This source code is licensed under the Attribution-NonCommercial 4.0 International License.
# You can find the license in the LICENSE file in the root directory of this source tree.
# --------------------------------------------------------

from __future__ import print_function

import torch
import torch.nn as nn
import math


class SimSiamLoss(nn.Module):
    def __init__(self):
        super(SimSiamLoss, self).__init__()
        self.criterion = nn.CosineSimilarity(dim=1)

    def forward(self, cl_features):

        if len(cl_features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, ...],'
                             'at least 3 dimensions are required')
        if len(cl_features.shape) > 3:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        cl_features_1 = cl_features[:, 0]  # [BS, feat_cl_dim]
        cl_features_2 = cl_features[:, 1]  # [BS, feat_cl_dim]
        loss = -(self.criterion(cl_features_1, cl_features_2).mean()) * 0.5

        # if not math.isfinite(loss):
        #     print(cl_features_1, '\n', cl_features_2)
        #     print(self.criterion(cl_features_1, cl_features_2))

        return loss


class BYOLLoss(nn.Module):
    def __init__(self):
        super(BYOLLoss, self).__init__()

    @staticmethod
    def forward(cl_features):

        if len(cl_features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, ...],'
                             'at least 3 dimensions are required')
        if len(cl_features.shape) > 3:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        cl_features_1 = cl_features[:, 0]  # [BS, feat_cl_dim]
        cl_features_2 = cl_features[:, 1]  # [BS, feat_cl_dim]
        loss = 2 - 2 * (cl_features_1 * cl_features_2).sum(dim=-1)
        # loss = 1 - (cl_features_1 * cl_features_2).sum(dim=-1)
        loss = loss.mean()

        if not math.isfinite(loss):
            print(cl_features_1, '\n', cl_features_2)
            print(2 - 2 * (cl_features_1 * cl_features_2).sum(dim=-1))

        return loss


# different implementation of InfoNCELoss, including MOCOV3Loss; SupConLoss
class InfoNCELoss(nn.Module):
    def __init__(self, temperature=0.1, contrast_sample='all'):
        """
        from CMAE: https://github.com/ZhichengHuang/CMAE/issues/5
        :param temperature: 0.1 0.5 1.0, 1.5 2.0
        """
        super(InfoNCELoss, self).__init__()
        self.temperature = temperature
        self.criterion = nn.CrossEntropyLoss()
        self.contrast_sample = contrast_sample

    def forward(self, cl_features):
        """
        Args:
            :param cl_features: : hidden vector of shape [bsz, n_views, ...]
        Returns:
            A loss scalar.
        """
        device = (torch.device('cuda')
                  if cl_features.is_cuda
                  else torch.device('cpu'))

        if len(cl_features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, ...],'
                             'at least 3 dimensions are required')
        if len(cl_features.shape) > 3:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        cl_features_1 = cl_features[:, 0]  # [BS, feat_cl_dim]
        cl_features_2 = cl_features[:, 1]  # [BS, feat_cl_dim]
        score_all = torch.matmul(cl_features_1, cl_features_2.transpose(1, 0))  # [BS, BS]
        score_all = score_all / self.temperature
        bs = score_all.size(0)

        if self.contrast_sample == 'all':
            score = score_all
        elif self.contrast_sample == 'positive':
            mask = torch.eye(bs, dtype=torch.float).to(device)  # torch.Size([BS, BS])
            score = score_all * mask
        else:
            raise ValueError('Contrastive sample: all{pos&neg} or positive(positive)')

        # label = (torch.arange(bs, dtype=torch.long) +
        #          bs * torch.distributed.get_rank()).to(device)
        label = torch.arange(bs, dtype=torch.long).to(device)

        loss = 2 * self.temperature * self.criterion(score, label)

        if not math.isfinite(loss):
            print(cl_features_1, '\n', cl_features_2)
            print(score_all, '\n', score, '\n', mask)

        return loss


class MOCOV3Loss(nn.Module):
    def __init__(self, temperature=0.1):
        super(MOCOV3Loss, self).__init__()
        self.temperature = temperature

    def forward(self, cl_features):

        if len(cl_features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, ...],'
                             'at least 3 dimensions are required')
        if len(cl_features.shape) > 3:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        cl_features_1 = cl_features[:, 0]  # [BS, feat_cl_dim]
        cl_features_2 = cl_features[:, 1]  # [BS, feat_cl_dim]

        # normalize
        cl_features_1 = nn.functional.normalize(cl_features_1, dim=1)
        cl_features_2 = nn.functional.normalize(cl_features_2, dim=1)
        # Einstein sum is more intuitive
        logits = torch.einsum('nc,mc->nm', [cl_features_1, cl_features_2]) / self.temperature
        N = logits.shape[0]
        labels = (torch.arange(N, dtype=torch.long)).cuda()
        return nn.CrossEntropyLoss()(logits, labels) * (2 * self.temperature)


class SupConLoss(nn.Module):
    """
    from: https://github.com/HobbitLong/SupContrast
    Supervised Contrastive Learning: https://arxiv.org/pdf/2004.11362.pdf.
    It also supports the unsupervised contrastive loss in SimCLR"""
    def __init__(self, temperature=0.1, contrast_mode='all', contrast_sample='all',
                 base_temperature=0.1):
        super(SupConLoss, self).__init__()
        self.temperature = temperature
        self.contrast_mode = contrast_mode
        self.contrast_sample = contrast_sample
        self.base_temperature = base_temperature

    def forward(self, features, labels=None, mask=None):
        """Compute loss for model. If both `labels` and `mask` are None,
        it degenerates to SimCLR unsupervised loss:
        https://arxiv.org/pdf/2002.05709.pdf

        Args:
            features: hidden vector of shape [bsz, n_views, ...].
            labels: ground truth of shape [bsz].
            mask: contrastive mask of shape [bsz, bsz], mask_{i,j}=1 if sample j
                has the same class as sample i. Can be asymmetric.
        Returns:
            A loss scalar.
        """
        device = (torch.device('cuda')
                  if features.is_cuda
                  else torch.device('cpu'))

        if len(features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, ...],'
                             'at least 3 dimensions are required')
        if len(features.shape) > 3:
            features = features.view(features.shape[0], features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        batch_size = features.shape[0]
        if labels is not None and mask is not None:
            raise ValueError('Cannot define both `labels` and `mask`')
        elif labels is None and mask is None:
            mask = torch.eye(batch_size, dtype=torch.float32).to(device)  # torch.Size([BS, BS])
        elif labels is not None:
            labels = labels.contiguous().view(-1, 1)
            if labels.shape[0] != batch_size:
                raise ValueError('Num of labels does not match num of features')
            mask = torch.eq(labels, labels.T).float().to(device)
        else:
            mask = mask.float().to(device)

        contrast_count = features.shape[1]  # contrast_count(2)
        contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0)  # [BS*contrast_count, D]
        if self.contrast_mode == 'one':
            anchor_feature = features[:, 0]  # [BS, D]
            anchor_count = 1
        elif self.contrast_mode == 'all':
            anchor_feature = contrast_feature  # [BS*contrast_count, D]
            anchor_count = contrast_count
        else:
            raise ValueError('Unknown mode: {}'.format(self.contrast_mode))

        # compute logits
        anchor_dot_contrast = torch.div(
            torch.matmul(anchor_feature, contrast_feature.T),
            self.temperature)  # [BS*contrast_count, BS*contrast_count]
        # for numerical stability
        logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True)  # [BS*contrast_count, 1]
        logits = anchor_dot_contrast - logits_max.detach()  # [BS*contrast_count, BS*contrast_count]

        # tile mask
        mask = mask.repeat(anchor_count, contrast_count)  # [BS*anchor_count, BS*contrast_count]
        # mask-out self-contrast cases
        logits_mask = torch.scatter(
            torch.ones_like(mask),
            1,
            torch.arange(batch_size * anchor_count).view(-1, 1).to(device),
            0
        )  # [BS*anchor_count, BS*contrast_count]
        mask = mask * logits_mask  # [BS*anchor_count, BS*contrast_count]

        """
        logits_mask is used to get the denominator(positives and negatives).
        mask is used to get the numerator(positives). mask is applied to log_prob.
        """

        # compute log_prob，logits_mask is contrast anchor with both positives and negatives
        exp_logits = torch.exp(logits) * logits_mask  # [BS*anchor_count, BS*contrast_count]
        # compute log_prob，logits_mask is contrast anchor with negatives, i.e., denominator only negatives contrast:
        # exp_logits = torch.exp(logits) * (logits_mask-mask)

        if self.contrast_sample == 'all':
            log_prob = logits - torch.log(exp_logits.sum(1, keepdim=True))  # [BS*anchor_count, BS*anchor_count]
            # compute mean of log-likelihood over positive
            mean_log_prob_pos = (mask * log_prob).sum(1) / mask.sum(1)  # [BS*anchor_count]
        elif self.contrast_sample == 'positive':
            mean_log_prob_pos = (mask * logits).sum(1) / mask.sum(1)
        else:
            raise ValueError('Contrastive sample: all{pos&neg} or positive(positive)')

        # loss
        loss = - (self.temperature / self.base_temperature) * mean_log_prob_pos
        loss = loss.view(anchor_count, batch_size).mean()

        return loss


class InfoNCELossPatchLevel(nn.Module):
    """
    test: ref ConMIM: https://github.com/TencentARC/ConMIM.
    """
    def __init__(self, temperature=0.1, contrast_sample='all'):
        """
        :param temperature: 0.1 0.5 1.0, 1.5 2.0
        """
        super(InfoNCELossPatchLevel, self).__init__()
        self.temperature = temperature
        self.criterion = nn.CrossEntropyLoss()
        self.contrast_sample = contrast_sample

        self.facial_region_group = [
            [2, 3],  # eyebrows
            [4, 5],  # eyes
            [6],  # nose
            [7, 8, 9],  # mouth
            [10, 1, 0],  # face boundaries
            [10],  # hair
            [1],  # facial skin
            [0]  # background
        ]

    def forward(self, cl_features, parsing_map=None):
        """
        Args:
            :param parsing_map:
            :param cl_features: : hidden vector of shape [bsz, n_views, ...]
        Returns:
            A loss scalar.
        """
        device = (torch.device('cuda')
                  if cl_features.is_cuda
                  else torch.device('cpu'))

        if len(cl_features.shape) < 4:
            raise ValueError('`features` needs to be [bsz, n_views, n_cl_patches, ...],'
                             'at least 4 dimensions are required')
        if len(cl_features.shape) > 4:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], cl_features.shape[2], -1)
            # [BS, 2, num_cl_patches, feat_cl_dim]

        cl_features_1 = cl_features[:, 0]
        cl_features_2 = cl_features[:, 1]
        score = torch.matmul(cl_features_1, cl_features_2.permute(0, 2, 1))  # [BS, num_cl_patches, num_cl_patches]
        score = score / self.temperature
        bs = score.size(0)
        num_cl_patches = score.size(1)

        if self.contrast_sample == 'all':
            score = score
        elif self.contrast_sample == 'positive':
            mask = torch.eye(num_cl_patches, dtype=torch.float32)  # torch.Size([num_cl_patches, num_cl_patches])
            mask_batch = mask.unsqueeze(0).expand(bs, -1).to(device)  # [bs, num_cl_patches, num_cl_patches]
            score = score*mask_batch
        elif self.contrast_sample == 'region':
            cl_features_1_fr = []
            cl_features_2_fr = []
            for facial_region_index in self.facial_region_group:
                fr_mask = (parsing_map == facial_region_index).unsqueeze(2).expand(-1, -1, cl_features_1.size(-1))
                cl_features_1_fr.append((cl_features_1 * fr_mask).mean(dim=1, keepdim=False))
                cl_features_2_fr.append((cl_features_1 * fr_mask).mean(dim=1, keepdim=False))
            cl_features_1_fr = torch.stack(cl_features_1_fr, dim=1)
            cl_features_2_fr = torch.stack(cl_features_2_fr, dim=1)
            score = torch.matmul(cl_features_1_fr, cl_features_2_fr.permute(0, 2, 1))  # [BS, 8, 8]
            score = score / self.temperature
            # mask = torch.eye(cl_features_1_fr.size(1), dtype=torch.bool)
            # torch.Size([cl_features_1_fr.size(1), cl_features_1_fr.size(1)])
            # mask_batch = mask.unsqueeze(0).expand(bs, -1).to(device)
            # [bs, cl_features_1_fr.size(1), cl_features_1_fr.size(1)]
            # score = score*mask_batch
            label = torch.arange(cl_features_1_fr.size(1), dtype=torch.long).to(device)
            labels_batch = label.unsqueeze(0).expand(bs, -1)
            loss = 2 * self.temperature * self.criterion(score, labels_batch)
            return loss
        else:
            raise ValueError('Contrastive sample: all{pos&neg} or positive(positive)')

        # label = (torch.arange(bs, dtype=torch.long) +
        #          bs * torch.distributed.get_rank()).to(device)
        label = torch.arange(num_cl_patches, dtype=torch.long).to(device)
        labels_batch = label.unsqueeze(0).expand(bs, -1)

        loss = 2 * self.temperature * self.criterion(score, labels_batch)

        return loss


class MSELoss(nn.Module):
    """
    test: unused
    """
    def __init__(self):
        super(MSELoss, self).__init__()

    @staticmethod
    def forward(cl_features):

        if len(cl_features.shape) < 3:
            raise ValueError('`features` needs to be [bsz, n_views, n_patches, ...],'
                             'at least 3 dimensions are required')
        if len(cl_features.shape) > 3:
            cl_features = cl_features.view(cl_features.shape[0], cl_features.shape[1], -1)  # [BS, 2, feat_cl_dim]

        cl_features_1 = cl_features[:, 0].float()  # [BS, feat_cl_dim]
        cl_features_2 = cl_features[:, 1].float()  # [BS, feat_cl_dim]

        return torch.nn.functional.mse_loss(cl_features_1, cl_features_2, reduction='mean')