FDD/label_distillation_rr.py
def train(train_loader, model, fixed_input, labels, criterion, labels_opt, epoch, classifier_weights, feat_opt,
          ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, adj_opt, adjust):
    """
    Do one epoch of training the synthetic labels.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    """
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    feat_losses = AverageMeter()
    top1 = AverageMeter()
    meta_loss_meter = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    w = classifier_weights.detach().clone()

    # define over how many steps to calculate the moving average
    # for resetting the model and also how many steps to wait
    # we use the same value for both
    if args.target == "cifar100" or args.target == "k49":
        stats_gap = 100
    elif args.num_base_examples > 100:
        stats_gap = 200
    else:
        stats_gap = 50

    for i, (input_, target) in enumerate(train_loader):
        # sample a minibatch of n_o target dataset examples x_t' with labels y_t'

        # measure data loading time
        data_time.update(time.time() - end)

        input_ = input_.to(device=device)
        target = target.to(device=device)

        # sample a minibatch of n_i examples from x~, y~: x~', y~'
        perm = torch.randperm(fixed_input.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = fixed_input[idx]
        lb_i = labels[idx]

        # solve the problem to find local weights w_l that minimize L(f_w_l(f_theta (x~')), y~')$
        feature = model(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)

        # update w <-- (1 - alpha) w + w_l (pseudo-gradient)
        w = (1 - 0.01) * w.detach() + 0.01 * w_local

        # find the logits for target data - using calibrated global weights
        logit = model(input_, adjust() * w)
        label_loss = criterion(logit, target)
        meta_loss_meter.update(label_loss.item(), input_.size(0))
        # update y~ <-- y~ - beta nabla_y~ L(f_w(f_theta(x_t)), y_t)

        labels_opt.zero_grad()
        label_loss.backward(retain_graph=True)
        labels_opt.step()

        # normalize the labels to form a valid probability distribution
        labels.data = torch.clamp(labels.data, 0, 1)
        labels.data = labels.data / labels.data.sum(dim=1).unsqueeze(1)

        # now update the model - using calibrated global weights
        # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~')
        feat_output = model(fi_i, adjust() * w)
        # ignore gradients from the labels
        feat_loss = soft_cross_entropy(feat_output, labels[idx].detach())

        # update the feature weights and also the calibration parameter
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()

        feat_losses.update(feat_loss.item(), input_.size(0))

        # measure accuracy and record loss
        err1 = compute_error_rate(logit.data, target, topk=(1,))[0]  # it returns a list

        losses.update(label_loss.item(), input_.size(0))
        top1.update(err1.item(), input_.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # update the moving average statistics
        if len(ma_list) < stats_gap:
            ma_list.append(err1.item())
            ma_sum += err1.item()
            current_num_steps += 1

            current_ma = ma_sum / len(ma_list)

            if current_num_steps == stats_gap:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
        else:
            ma_sum = ma_sum - ma_list[0] + err1.item()
            ma_list = ma_list[1:] + [err1.item()]
            current_num_steps += 1

            current_ma = ma_sum / len(ma_list)

            if ma_sum < lowest_ma_sum:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
            elif num_steps_from_min < stats_gap:
                num_steps_from_min += 1
            else:
                # do early stopping
                num_steps_list.append(
                    current_num_steps - num_steps_from_min - 1)
                # restart all metrics
                ma_list = []
                ma_sum = 0
                lowest_ma_sum = 999999999
                current_num_steps = 0
                num_steps_from_min = 0

                # restart the model and the optimizers
                if args.target == "cifar10" or args.target == "cifar100":
                    if args.resnet:
                        model = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                       bottleneck=False).to(device=device)
                    else:
                        model = AlexCifarNet(args).to(device=device)
                else:
                    model = LeNet(args).to(device=device)
                feat_opt = torch.optim.Adam(model.parameters())
                adjust = AdjustLayer(device=device)
                adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)

                w = torch.zeros(
                    num_out_feats + 1, num_classes, device=device, requires_grad=False)

                print('Model restarted after ' +
                      str(num_steps_list[-1]) + ' steps')

        if i % args.print_freq == 0 and args.verbose is True:
            print('Epoch: [{0}/{1}][{2}/{3}]\t'
                  'LR: {LR:.6f}\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t'
                  'MAvg 1-err {current_ma:.4f}'.format(
                      epoch, args.epochs, i, len(train_loader), LR=1, batch_time=batch_time,
                      data_time=data_time, loss=losses, top1=top1, current_ma=current_ma))

    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}'.format(
        epoch, args.epochs, top1=top1, loss=losses))

    return top1.avg, losses.avg, labels, meta_loss_meter.avg, w, feat_losses.avg, \
        ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, \
        model, feat_opt, adj_opt, adjust

    Do one epoch of training the synthetic labels.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    
Do one epoch of training the synthetic labels.
Do one epoch of training the synthetic labels .
def train(train_loader, model, fixed_input, labels, criterion, labels_opt, epoch, classifier_weights, feat_opt,
          ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, adj_opt, adjust):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    feat_losses = AverageMeter()
    top1 = AverageMeter()
    meta_loss_meter = AverageMeter()
    model.train()
    end = time.time()
    w = classifier_weights.detach().clone()
    if args.target == "cifar100" or args.target == "k49":
        stats_gap = 100
    elif args.num_base_examples > 100:
        stats_gap = 200
    else:
        stats_gap = 50
    for i, (input_, target) in enumerate(train_loader):
        data_time.update(time.time() - end)
        input_ = input_.to(device=device)
        target = target.to(device=device)
        perm = torch.randperm(fixed_input.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = fixed_input[idx]
        lb_i = labels[idx]
        feature = model(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)
        w = (1 - 0.01) * w.detach() + 0.01 * w_local
        logit = model(input_, adjust() * w)
        label_loss = criterion(logit, target)
        meta_loss_meter.update(label_loss.item(), input_.size(0))
        labels_opt.zero_grad()
        label_loss.backward(retain_graph=True)
        labels_opt.step()
        labels.data = torch.clamp(labels.data, 0, 1)
        labels.data = labels.data / labels.data.sum(dim=1).unsqueeze(1)
        feat_output = model(fi_i, adjust() * w)
        feat_loss = soft_cross_entropy(feat_output, labels[idx].detach())
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()
        feat_losses.update(feat_loss.item(), input_.size(0))
        err1 = compute_error_rate(logit.data, target, topk=(1,))[0]  
        losses.update(label_loss.item(), input_.size(0))
        top1.update(err1.item(), input_.size(0))
        batch_time.update(time.time() - end)
        end = time.time()
        if len(ma_list) < stats_gap:
            ma_list.append(err1.item())
            ma_sum += err1.item()
            current_num_steps += 1
            current_ma = ma_sum / len(ma_list)
            if current_num_steps == stats_gap:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
        else:
            ma_sum = ma_sum - ma_list[0] + err1.item()
            ma_list = ma_list[1:] + [err1.item()]
            current_num_steps += 1
            current_ma = ma_sum / len(ma_list)
            if ma_sum < lowest_ma_sum:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
            elif num_steps_from_min < stats_gap:
                num_steps_from_min += 1
            else:
                num_steps_list.append(
                    current_num_steps - num_steps_from_min - 1)
                ma_list = []
                ma_sum = 0
                lowest_ma_sum = 999999999
                current_num_steps = 0
                num_steps_from_min = 0
                if args.target == "cifar10" or args.target == "cifar100":
                    if args.resnet:
                        model = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                       bottleneck=False).to(device=device)
                    else:
                        model = AlexCifarNet(args).to(device=device)
                else:
                    model = LeNet(args).to(device=device)
                feat_opt = torch.optim.Adam(model.parameters())
                adjust = AdjustLayer(device=device)
                adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)
                w = torch.zeros(
                    num_out_feats + 1, num_classes, device=device, requires_grad=False)
                print('Model restarted after ' +
                      str(num_steps_list[-1]) + ' steps')
        if i % args.print_freq == 0 and args.verbose is True:
            print('Epoch: [{0}/{1}][{2}/{3}]\t'
                  'LR: {LR:.6f}\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t'
                  'MAvg 1-err {current_ma:.4f}'.format(
                      epoch, args.epochs, i, len(train_loader), LR=1, batch_time=batch_time,
                      data_time=data_time, loss=losses, top1=top1, current_ma=current_ma))
    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}'.format(
        epoch, args.epochs, top1=top1, loss=losses))
    return top1.avg, losses.avg, labels, meta_loss_meter.avg, w, feat_losses.avg, \
        ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, \
        model, feat_opt, adj_opt, adjust
def train ( train_loader , model , fixed_input , labels , criterion , labels_opt , epoch , classifier_weights , feat_opt , ma_list , ma_sum , lowest_ma_sum , current_num_steps , num_steps_list , num_steps_from_min , adj_opt , adjust ) : batch_time = AverageMeter ( ) data_time = AverageMeter ( ) losses = AverageMeter ( ) feat_losses = AverageMeter ( ) top1 = AverageMeter ( ) meta_loss_meter = AverageMeter ( ) model . train ( ) end = time . time ( ) w = classifier_weights . detach ( ) . clone ( ) if args . target == "cifar100" or args . target == "k49" : stats_gap = 100 elif args . num_base_examples > 100 : stats_gap = 200 else : stats_gap = 50 for i , ( input_ , target ) in enumerate ( train_loader ) : data_time . update ( time . time ( ) - end ) input_ = input_ . to ( device = device ) target = target . to ( device = device ) perm = torch . randperm ( fixed_input . size ( 0 ) ) idx = perm [ : args . inner_batch_size ] fi_i = fixed_input [ idx ] lb_i = labels [ idx ] feature = model ( fi_i ) feature_add_bias = torch . cat ( ( feature , torch . ones ( ( args . inner_batch_size , 1 ) , device = device ) ) , 1 ) w_local = get_local_weights ( feature_add_bias , lb_i , device ) w = ( 1 - 0.01 ) * w . detach ( ) + 0.01 * w_local logit = model ( input_ , adjust ( ) * w ) label_loss = criterion ( logit , target ) meta_loss_meter . update ( label_loss . item ( ) , input_ . size ( 0 ) ) labels_opt . zero_grad ( ) label_loss . backward ( retain_graph = True ) labels_opt . step ( ) labels . data = torch . clamp ( labels . data , 0 , 1 ) labels . data = labels . data / labels . data . sum ( dim = 1 ) . unsqueeze ( 1 ) feat_output = model ( fi_i , adjust ( ) * w ) feat_loss = soft_cross_entropy ( feat_output , labels [ idx ] . detach ( ) ) feat_opt . zero_grad ( ) adj_opt . zero_grad ( ) feat_loss . backward ( ) feat_opt . step ( ) adj_opt . step ( ) feat_losses . update ( feat_loss . item ( ) , input_ . size ( 0 ) ) err1 = compute_error_rate ( logit . data , target , topk = ( 1 , ) ) [ 0 ] losses . update ( label_loss . item ( ) , input_ . size ( 0 ) ) top1 . update ( err1 . item ( ) , input_ . size ( 0 ) ) batch_time . update ( time . time ( ) - end ) end = time . time ( ) if len ( ma_list ) < stats_gap : ma_list . append ( err1 . item ( ) ) ma_sum += err1 . item ( ) current_num_steps += 1 current_ma = ma_sum / len ( ma_list ) if current_num_steps == stats_gap : lowest_ma_sum = ma_sum num_steps_from_min = 0 else : ma_sum = ma_sum - ma_list [ 0 ] + err1 . item ( ) ma_list = ma_list [ 1 : ] + [ err1 . item ( ) ] current_num_steps += 1 current_ma = ma_sum / len ( ma_list ) if ma_sum < lowest_ma_sum : lowest_ma_sum = ma_sum num_steps_from_min = 0 elif num_steps_from_min < stats_gap : num_steps_from_min += 1 else : num_steps_list . append ( current_num_steps - num_steps_from_min - 1 ) ma_list = [ ] ma_sum = 0 lowest_ma_sum = 999999999 current_num_steps = 0 num_steps_from_min = 0 if args . target == "cifar10" or args . target == "cifar100" : if args . resnet : model = ResNet ( dataset = args . target , depth = 18 , num_classes = num_classes , bottleneck = False ) . to ( device = device ) else : model = AlexCifarNet ( args ) . to ( device = device ) else : model = LeNet ( args ) . to ( device = device ) feat_opt = torch . optim . Adam ( model . parameters ( ) ) adjust = AdjustLayer ( device = device ) adj_opt = torch . optim . Adam ( list ( adjust . parameters ( ) ) , 0.01 ) w = torch . zeros ( num_out_feats + 1 , num_classes , device = device , requires_grad = False ) print ( 'Model restarted after ' + str ( num_steps_list [ - 1 ] ) + ' steps' ) if i % args . print_freq == 0 and args . verbose is True : print ( 'Epoch: [{0}/{1}][{2}/{3}]\t' 'LR: {LR:.6f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t' 'MAvg 1-err {current_ma:.4f}' . format ( epoch , args . epochs , i , len ( train_loader ) , LR = 1 , batch_time = batch_time , data_time = data_time , loss = losses , top1 = top1 , current_ma = current_ma ) ) print ( '* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}' . format ( epoch , args . epochs , top1 = top1 , loss = losses ) ) return top1 . avg , losses . avg , labels , meta_loss_meter . avg , w , feat_losses . avg , ma_list , ma_sum , lowest_ma_sum , current_num_steps , num_steps_list , num_steps_from_min , model , feat_opt , adj_opt , adjust
Do one epoch of training the synthetic labels.
Do one epoch of training the synthetic labels .
"""
    Do one epoch of training the synthetic labels.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    """ # switch to train mode # define over how many steps to calculate the moving average # for resetting the model and also how many steps to wait # we use the same value for both # sample a minibatch of n_o target dataset examples x_t' with labels y_t' # measure data loading time # sample a minibatch of n_i examples from x~, y~: x~', y~' # solve the problem to find local weights w_l that minimize L(f_w_l(f_theta (x~')), y~')$ # update w <-- (1 - alpha) w + w_l (pseudo-gradient) # find the logits for target data - using calibrated global weights # update y~ <-- y~ - beta nabla_y~ L(f_w(f_theta(x_t)), y_t) # normalize the labels to form a valid probability distribution # now update the model - using calibrated global weights # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~') # ignore gradients from the labels # update the feature weights and also the calibration parameter # measure accuracy and record loss # it returns a list # measure elapsed time # update the moving average statistics # do early stopping # restart all metrics # restart the model and the optimizers
train_loader
model
fixed_input
labels
criterion
labels_opt
epoch
classifier_weights
feat_opt
ma_list
ma_sum
lowest_ma_sum
current_num_steps
num_steps_list
num_steps_from_min
adj_opt
adjust