FDD/label_distillation_rr.py
def find_best_num_steps(val_loader, criterion, fixed_input, labels):
    """Calculate the best number of steps to use based on the validation set."""
    best_num_steps = 0
    lowest_err = 100
    errors_list = []
    num_steps_used = []

    # use a larger number of max steps when using more base examples
    if args.num_base_examples > 100:
        max_num_steps = 1701
    else:
        max_num_steps = 1000

    # initialize a new model
    if args.target == "cifar10" or args.target == "cifar100":
        if args.resnet:
            model_eval = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                bottleneck=False).to(device=device)
        else:
            model_eval = AlexCifarNet(args).to(device=device)
    else:
        model_eval = LeNet(args).to(device=device)
    feat_opt = torch.optim.Adam(model_eval.parameters())
    adjust = AdjustLayer(device=device)
    adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)
    model_eval.train()

    w_eval = torch.zeros(num_out_feats + 1, num_classes,
                         device=device, requires_grad=False)

    for ITER in range(max_num_steps):
        # sample a minibatch of n_i examples from x~, y~: x~', y~'
        perm = torch.randperm(fixed_input.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = fixed_input[idx]
        lb_i = labels[idx]

        feature = model_eval(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)
        # update w <-- alpha w + (1-alpha) w_l (pseudo-gradient)
        w_eval = (1 - 0.01) * w_eval.detach() + 0.01 * w_local

        # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~')
        feat_output = model_eval(fi_i, adjust() * w_eval)

        feat_loss = soft_cross_entropy(feat_output, labels[idx].detach())
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()

        # for larger numbers of base examples we decrease the frequency of evaluation
        if args.num_base_examples > 100:
            if ITER in set([9, 24, 74, 100, 200, 300, 500, 700, 1000, 1200, 1500, 1700]):
                validate_now = True
            else:
                validate_now = False
        else:
            if ITER % 50 == 49 or ITER in set([9, 24, 74]):
                validate_now = True
            else:
                validate_now = False

        if validate_now:
            # switch to evaluate mode
            model_eval.eval()
            top1_eval = AverageMeter()
            losses_eval = AverageMeter()

            for i, (input_, target) in enumerate(val_loader):
                input_ = input_.to(device=device)
                target = target.to(device=device)
                output = model_eval(input_, w_eval)
                loss = criterion(output, target)

                # measure error and record loss
                err1 = compute_error_rate(output.data, target, topk=(1,))[0]
                top1_eval.update(err1.item(), input_.size(0))
                losses_eval.update(loss.item(), input_.size(0))

            errors_list.append(top1_eval.avg)
            num_steps_used.append(ITER + 1)
            if top1_eval.avg < lowest_err:
                lowest_err = top1_eval.avg
                best_num_steps = ITER + 1

    print(num_steps_used)
    print(errors_list)
    return best_num_steps, errors_list, num_steps_used
Calculate the best number of steps to use based on the validation set.
Calculate the best number of steps to use based on the validation set.
Calculate the best number of steps to use based on the validation set .
def find_best_num_steps(val_loader, criterion, fixed_input, labels):
    best_num_steps = 0
    lowest_err = 100
    errors_list = []
    num_steps_used = []
    if args.num_base_examples > 100:
        max_num_steps = 1701
    else:
        max_num_steps = 1000
    if args.target == "cifar10" or args.target == "cifar100":
        if args.resnet:
            model_eval = ResNet(dataset=args.target, depth=18, num_classes=num_classes,
                                bottleneck=False).to(device=device)
        else:
            model_eval = AlexCifarNet(args).to(device=device)
    else:
        model_eval = LeNet(args).to(device=device)
    feat_opt = torch.optim.Adam(model_eval.parameters())
    adjust = AdjustLayer(device=device)
    adj_opt = torch.optim.Adam(list(adjust.parameters()), 0.01)
    model_eval.train()
    w_eval = torch.zeros(num_out_feats + 1, num_classes,
                         device=device, requires_grad=False)
    for ITER in range(max_num_steps):
        perm = torch.randperm(fixed_input.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = fixed_input[idx]
        lb_i = labels[idx]
        feature = model_eval(fi_i)
        feature_add_bias = torch.cat(
            (feature, torch.ones((args.inner_batch_size, 1), device=device)), 1)
        w_local = get_local_weights(feature_add_bias, lb_i, device)
        w_eval = (1 - 0.01) * w_eval.detach() + 0.01 * w_local
        feat_output = model_eval(fi_i, adjust() * w_eval)
        feat_loss = soft_cross_entropy(feat_output, labels[idx].detach())
        feat_opt.zero_grad()
        adj_opt.zero_grad()
        feat_loss.backward()
        feat_opt.step()
        adj_opt.step()
        if args.num_base_examples > 100:
            if ITER in set([9, 24, 74, 100, 200, 300, 500, 700, 1000, 1200, 1500, 1700]):
                validate_now = True
            else:
                validate_now = False
        else:
            if ITER % 50 == 49 or ITER in set([9, 24, 74]):
                validate_now = True
            else:
                validate_now = False
        if validate_now:
            model_eval.eval()
            top1_eval = AverageMeter()
            losses_eval = AverageMeter()
            for i, (input_, target) in enumerate(val_loader):
                input_ = input_.to(device=device)
                target = target.to(device=device)
                output = model_eval(input_, w_eval)
                loss = criterion(output, target)
                err1 = compute_error_rate(output.data, target, topk=(1,))[0]
                top1_eval.update(err1.item(), input_.size(0))
                losses_eval.update(loss.item(), input_.size(0))
            errors_list.append(top1_eval.avg)
            num_steps_used.append(ITER + 1)
            if top1_eval.avg < lowest_err:
                lowest_err = top1_eval.avg
                best_num_steps = ITER + 1
    print(num_steps_used)
    print(errors_list)
    return best_num_steps, errors_list, num_steps_used
def find_best_num_steps ( val_loader , criterion , fixed_input , labels ) : best_num_steps = 0 lowest_err = 100 errors_list = [ ] num_steps_used = [ ] if args . num_base_examples > 100 : max_num_steps = 1701 else : max_num_steps = 1000 if args . target == "cifar10" or args . target == "cifar100" : if args . resnet : model_eval = ResNet ( dataset = args . target , depth = 18 , num_classes = num_classes , bottleneck = False ) . to ( device = device ) else : model_eval = AlexCifarNet ( args ) . to ( device = device ) else : model_eval = LeNet ( args ) . to ( device = device ) feat_opt = torch . optim . Adam ( model_eval . parameters ( ) ) adjust = AdjustLayer ( device = device ) adj_opt = torch . optim . Adam ( list ( adjust . parameters ( ) ) , 0.01 ) model_eval . train ( ) w_eval = torch . zeros ( num_out_feats + 1 , num_classes , device = device , requires_grad = False ) for ITER in range ( max_num_steps ) : perm = torch . randperm ( fixed_input . size ( 0 ) ) idx = perm [ : args . inner_batch_size ] fi_i = fixed_input [ idx ] lb_i = labels [ idx ] feature = model_eval ( fi_i ) feature_add_bias = torch . cat ( ( feature , torch . ones ( ( args . inner_batch_size , 1 ) , device = device ) ) , 1 ) w_local = get_local_weights ( feature_add_bias , lb_i , device ) w_eval = ( 1 - 0.01 ) * w_eval . detach ( ) + 0.01 * w_local feat_output = model_eval ( fi_i , adjust ( ) * w_eval ) feat_loss = soft_cross_entropy ( feat_output , labels [ idx ] . detach ( ) ) feat_opt . zero_grad ( ) adj_opt . zero_grad ( ) feat_loss . backward ( ) feat_opt . step ( ) adj_opt . step ( ) if args . num_base_examples > 100 : if ITER in set ( [ 9 , 24 , 74 , 100 , 200 , 300 , 500 , 700 , 1000 , 1200 , 1500 , 1700 ] ) : validate_now = True else : validate_now = False else : if ITER % 50 == 49 or ITER in set ( [ 9 , 24 , 74 ] ) : validate_now = True else : validate_now = False if validate_now : model_eval . eval ( ) top1_eval = AverageMeter ( ) losses_eval = AverageMeter ( ) for i , ( input_ , target ) in enumerate ( val_loader ) : input_ = input_ . to ( device = device ) target = target . to ( device = device ) output = model_eval ( input_ , w_eval ) loss = criterion ( output , target ) err1 = compute_error_rate ( output . data , target , topk = ( 1 , ) ) [ 0 ] top1_eval . update ( err1 . item ( ) , input_ . size ( 0 ) ) losses_eval . update ( loss . item ( ) , input_ . size ( 0 ) ) errors_list . append ( top1_eval . avg ) num_steps_used . append ( ITER + 1 ) if top1_eval . avg < lowest_err : lowest_err = top1_eval . avg best_num_steps = ITER + 1 print ( num_steps_used ) print ( errors_list ) return best_num_steps , errors_list , num_steps_used
Calculate the best number of steps to use based on the validation set.
Calculate the best number of steps to use based on the validation set .
"""Calculate the best number of steps to use based on the validation set.""" # use a larger number of max steps when using more base examples # initialize a new model # sample a minibatch of n_i examples from x~, y~: x~', y~' # update w <-- alpha w + (1-alpha) w_l (pseudo-gradient) # update theta <-- theta - beta nabla_theta L(f_w(f_theta(x~')), y~') # for larger numbers of base examples we decrease the frequency of evaluation # switch to evaluate mode # measure error and record loss
val_loader
criterion
fixed_input
labels