FDD/data_distillation_rr.py
def validate(val_loader, model, criterion, epoch, distill_data, distill_labels, num_steps, normalize):
    """Validation phase. This involves training a model from scratch."""
    losses_eval = AverageMeter()
    top1_eval = AverageMeter()
    start = time.time()

    print('Number of steps for retraining during validation: ' + str(num_steps))

    # initialize a new model
    if args.target == "cifar10":
        if args.resnet:
            model_eval = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                bottleneck=False).to(device=device)
        else:
            model_eval = AlexCifarNet(args).to(device=device)
    else:
        model_eval = LeNet(args).to(device=device)
    feat_opt = torch.optim.Adam(model_eval.parameters())
    adjust = AdjustLayer(device=device)
    adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)
    model_eval.train()

    w_eval = torch.zeros(num_out_feats + 1, num_classes,
                         device=device, requires_grad=False)

    # train a model from scratch for the given number of steps
    # using only the synthetic images and their labels
    for ITER in range(num_steps):
        # sample a minibatch of n_i examples from x~, y~: x~', y~'
        perm = torch.randperm(distill_data.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].detach().cpu()]).to(device=device)
        lb_i = distill_labels[idx]

        # solve the problem to find local weights w_l that minimize L(f_w_l(f_theta (x~')), y~')$
        feature = model_eval(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)

        # update w <-- (1 - alpha) w + alpha w_l (pseudo-gradient)
        w_eval = (1 - 0.01) * w_eval.detach() + 0.01 * w_local

        # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~')
        # also update the calibration
        feat_output = model_eval(fi_i.detach(), adjust() * w_eval)
        feat_loss = soft_cross_entropy(feat_output, lb_i)
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()

    # evaluate the validation error
    # switch to evaluate mode
    model_eval.eval()

    for i, (input_, target) in enumerate(val_loader):
        input_ = input_.to(device=device)
        target = target.to(device=device)
        output = model_eval(input_, w_eval)
        loss = criterion(output, target)

        # measure error rate and record loss
        err1 = compute_error_rate(output.data, target, topk=(1,))[0]
        top1_eval.update(err1.item(), input_.size(0))
        losses_eval.update(loss.item(), input_.size(0))

    val_time = time.time() - start
    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f} Val Loss {loss.avg:.3f} Time {val_time:.3f}'.format(
        epoch, args.epochs, top1=top1_eval, loss=losses_eval, val_time=val_time))

    return top1_eval.avg, losses_eval.avg
Validation phase. This involves training a model from scratch.
Validation phase. This involves training a model from scratch.
Validation phase . This involves training a model from scratch .
def validate(val_loader, model, criterion, epoch, distill_data, distill_labels, num_steps, normalize):
    losses_eval = AverageMeter()
    top1_eval = AverageMeter()
    start = time.time()
    print('Number of steps for retraining during validation: ' + str(num_steps))
    if args.target == "cifar10":
        if args.resnet:
            model_eval = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                bottleneck=False).to(device=device)
        else:
            model_eval = AlexCifarNet(args).to(device=device)
    else:
        model_eval = LeNet(args).to(device=device)
    feat_opt = torch.optim.Adam(model_eval.parameters())
    adjust = AdjustLayer(device=device)
    adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)
    model_eval.train()
    w_eval = torch.zeros(num_out_feats + 1, num_classes,
                         device=device, requires_grad=False)
    for ITER in range(num_steps):
        perm = torch.randperm(distill_data.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].detach().cpu()]).to(device=device)
        lb_i = distill_labels[idx]
        feature = model_eval(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)
        w_eval = (1 - 0.01) * w_eval.detach() + 0.01 * w_local
        feat_output = model_eval(fi_i.detach(), adjust() * w_eval)
        feat_loss = soft_cross_entropy(feat_output, lb_i)
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()
    model_eval.eval()
    for i, (input_, target) in enumerate(val_loader):
        input_ = input_.to(device=device)
        target = target.to(device=device)
        output = model_eval(input_, w_eval)
        loss = criterion(output, target)
        err1 = compute_error_rate(output.data, target, topk=(1,))[0]
        top1_eval.update(err1.item(), input_.size(0))
        losses_eval.update(loss.item(), input_.size(0))
    val_time = time.time() - start
    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f} Val Loss {loss.avg:.3f} Time {val_time:.3f}'.format(
        epoch, args.epochs, top1=top1_eval, loss=losses_eval, val_time=val_time))
    return top1_eval.avg, losses_eval.avg
def validate ( val_loader , model , criterion , epoch , distill_data , distill_labels , num_steps , normalize ) : losses_eval = AverageMeter ( ) top1_eval = AverageMeter ( ) start = time . time ( ) print ( 'Number of steps for retraining during validation: ' + str ( num_steps ) ) if args . target == "cifar10" : if args . resnet : model_eval = ResNet ( dataset = args . target , depth = 18 , num_classes = num_classes , bottleneck = False ) . to ( device = device ) else : model_eval = AlexCifarNet ( args ) . to ( device = device ) else : model_eval = LeNet ( args ) . to ( device = device ) feat_opt = torch . optim . Adam ( model_eval . parameters ( ) ) adjust = AdjustLayer ( device = device ) adj_opt = torch . optim . Adam ( list ( adjust . parameters ( ) ) , 0.01 ) model_eval . train ( ) w_eval = torch . zeros ( num_out_feats + 1 , num_classes , device = device , requires_grad = False ) for ITER in range ( num_steps ) : perm = torch . randperm ( distill_data . size ( 0 ) ) idx = perm [ : args . inner_batch_size ] fi_i = torch . stack ( [ normalize ( image ) for image in distill_data [ idx ] . detach ( ) . cpu ( ) ] ) . to ( device = device ) lb_i = distill_labels [ idx ] feature = model_eval ( fi_i ) feature_add_bias = torch . cat ( ( feature , torch . ones ( ( args . inner_batch_size , 1 ) , device = device ) ) , 1 ) w_local = get_local_weights ( feature_add_bias , lb_i , device ) w_eval = ( 1 - 0.01 ) * w_eval . detach ( ) + 0.01 * w_local feat_output = model_eval ( fi_i . detach ( ) , adjust ( ) * w_eval ) feat_loss = soft_cross_entropy ( feat_output , lb_i ) feat_opt . zero_grad ( ) adj_opt . zero_grad ( ) feat_loss . backward ( ) feat_opt . step ( ) adj_opt . step ( ) model_eval . eval ( ) for i , ( input_ , target ) in enumerate ( val_loader ) : input_ = input_ . to ( device = device ) target = target . to ( device = device ) output = model_eval ( input_ , w_eval ) loss = criterion ( output , target ) err1 = compute_error_rate ( output . data , target , topk = ( 1 , ) ) [ 0 ] top1_eval . update ( err1 . item ( ) , input_ . size ( 0 ) ) losses_eval . update ( loss . item ( ) , input_ . size ( 0 ) ) val_time = time . time ( ) - start print ( '* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f} Val Loss {loss.avg:.3f} Time {val_time:.3f}' . format ( epoch , args . epochs , top1 = top1_eval , loss = losses_eval , val_time = val_time ) ) return top1_eval . avg , losses_eval . avg
Validation phase.
Validation phase .
"""Validation phase. This involves training a model from scratch.""" # initialize a new model # train a model from scratch for the given number of steps # using only the synthetic images and their labels # sample a minibatch of n_i examples from x~, y~: x~', y~' # solve the problem to find local weights w_l that minimize L(f_w_l(f_theta (x~')), y~')$ # update w <-- (1 - alpha) w + alpha w_l (pseudo-gradient) # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~') # also update the calibration # evaluate the validation error # switch to evaluate mode # measure error rate and record loss
val_loader
model
criterion
epoch
distill_data
distill_labels
num_steps
normalize