FDD/data_distillation_or2.py
def train(train_loader, model, distill_data, distill_labels, criterion, data_opt, epoch, optimizer,
          ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, normalize):
    """
    Do one epoch of training the synthetic images.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    """
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    model_losses = AverageMeter()
    top1 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()

    for i, (input_, target) in enumerate(train_loader):
        # sample a minibatch of n_o target dataset examples x_t' with labels y_t'

        # measure data loading time
        data_time.update(time.time() - end)

        input_ = input_.to(device=device)
        target = target.to(device=device)

        # sample a minibatch of n_i examples from x~, y~: x~', y~'
        perm = torch.randperm(distill_data.size(0))
        idx = perm[:args.inner_batch_size]
        # normalize the synthetic images using the standard normalization
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].cpu()]).to(device=device)
        lb_i = distill_labels[idx]

        # inner loop
        for weight in model.parameters():
            weight.fast = None
        fi_o = model(fi_i)
        loss = soft_cross_entropy(fi_o, lb_i)
        optimizer.zero_grad()
        grad = torch.autograd.grad(loss, model.parameters(), create_graph=True)
        # create fast weights so that we can use second-order gradient
        for k, weight in enumerate(model.parameters()):
            weight.fast = weight - args.inner_lr * grad[k]

        # outer loop
        # update x~ <-- x~ - beta nabla_x~ L(f_w(f_theta(x_t)), y_t)
        # fast weights will be used
        logit = model(input_)
        data_loss = criterion(logit, target)
        data_opt.zero_grad()
        data_loss.backward(retain_graph=False)
        data_opt.step()

        # make sure the synthetic examples have valid values after the update
        distill_data.data = torch.clamp(distill_data.data, 0, 1)

        # calculate the loss again for updating the features
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].cpu()]).to(device=device)
        lb_i = distill_labels[idx]
        for weight in model.parameters():
            weight.fast = None
        fi_o = model(fi_i)
        loss = soft_cross_entropy(fi_o, lb_i)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        model_losses.update(loss.item(), input_.size(0))

        # measure error rate and record loss
        err1 = compute_error_rate(logit.data, target, topk=(1,))[0]  # it returns a list

        losses.update(data_loss.item(), input_.size(0))
        top1.update(err1.item(), input_.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # update the moving average statistics
        # for image distillation, the values are fixed to 50
        if len(ma_list) < 50:
            ma_list.append(err1.item())
            ma_sum += err1.item()
            current_num_steps += 1

            current_ma = ma_sum / len(ma_list)

            if current_num_steps == 50:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
        else:
            ma_sum = ma_sum - ma_list[0] + err1.item()
            ma_list = ma_list[1:] + [err1.item()]
            current_num_steps += 1

            current_ma = ma_sum / len(ma_list)

            if ma_sum < lowest_ma_sum:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
            elif num_steps_from_min < 50:
                num_steps_from_min += 1
            else:
                # do early stopping
                num_steps_list.append(
                    current_num_steps - num_steps_from_min - 1)
                # restart all metrics
                ma_list = []
                ma_sum = 0
                lowest_ma_sum = 999999999
                current_num_steps = 0
                num_steps_from_min = 0

                # restart the model and the optimizer
                if args.target == "cifar10":
                    model = M.AlexCifarNetMeta(args).to(device=device)
                else:
                    model = M.LeNetMeta(args).to(device=device)
                optimizer = torch.optim.Adam(model.parameters())

                print('Model restarted after ' +
                      str(num_steps_list[-1]) + ' steps')

        if i % args.print_freq == 0 and args.verbose is True:
            print('Epoch: [{0}/{1}][{2}/{3}]\t'
                  'LR: {LR:.6f}\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t'
                  'MAvg 1-err {current_ma:.4f}'.format(
                      epoch, args.epochs, i, len(train_loader), LR=1, batch_time=batch_time,
                      data_time=data_time, loss=losses, top1=top1, current_ma=current_ma))

    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}'.format(
        epoch, args.epochs, top1=top1, loss=losses))

    return top1.avg, losses.avg, distill_data, model_losses.avg, \
        ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, \
        model, optimizer

    Do one epoch of training the synthetic images.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    
Do one epoch of training the synthetic images.
Do one epoch of training the synthetic images .
def train(train_loader, model, distill_data, distill_labels, criterion, data_opt, epoch, optimizer,
          ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, normalize):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    model_losses = AverageMeter()
    top1 = AverageMeter()
    model.train()
    end = time.time()
    for i, (input_, target) in enumerate(train_loader):
        data_time.update(time.time() - end)
        input_ = input_.to(device=device)
        target = target.to(device=device)
        perm = torch.randperm(distill_data.size(0))
        idx = perm[:args.inner_batch_size]
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].cpu()]).to(device=device)
        lb_i = distill_labels[idx]
        for weight in model.parameters():
            weight.fast = None
        fi_o = model(fi_i)
        loss = soft_cross_entropy(fi_o, lb_i)
        optimizer.zero_grad()
        grad = torch.autograd.grad(loss, model.parameters(), create_graph=True)
        for k, weight in enumerate(model.parameters()):
            weight.fast = weight - args.inner_lr * grad[k]
        logit = model(input_)
        data_loss = criterion(logit, target)
        data_opt.zero_grad()
        data_loss.backward(retain_graph=False)
        data_opt.step()
        distill_data.data = torch.clamp(distill_data.data, 0, 1)
        fi_i = torch.stack([normalize(image)
                            for image in distill_data[idx].cpu()]).to(device=device)
        lb_i = distill_labels[idx]
        for weight in model.parameters():
            weight.fast = None
        fi_o = model(fi_i)
        loss = soft_cross_entropy(fi_o, lb_i)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        model_losses.update(loss.item(), input_.size(0))
        err1 = compute_error_rate(logit.data, target, topk=(1,))[0]  
        losses.update(data_loss.item(), input_.size(0))
        top1.update(err1.item(), input_.size(0))
        batch_time.update(time.time() - end)
        end = time.time()
        if len(ma_list) < 50:
            ma_list.append(err1.item())
            ma_sum += err1.item()
            current_num_steps += 1
            current_ma = ma_sum / len(ma_list)
            if current_num_steps == 50:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
        else:
            ma_sum = ma_sum - ma_list[0] + err1.item()
            ma_list = ma_list[1:] + [err1.item()]
            current_num_steps += 1
            current_ma = ma_sum / len(ma_list)
            if ma_sum < lowest_ma_sum:
                lowest_ma_sum = ma_sum
                num_steps_from_min = 0
            elif num_steps_from_min < 50:
                num_steps_from_min += 1
            else:
                num_steps_list.append(
                    current_num_steps - num_steps_from_min - 1)
                ma_list = []
                ma_sum = 0
                lowest_ma_sum = 999999999
                current_num_steps = 0
                num_steps_from_min = 0
                if args.target == "cifar10":
                    model = M.AlexCifarNetMeta(args).to(device=device)
                else:
                    model = M.LeNetMeta(args).to(device=device)
                optimizer = torch.optim.Adam(model.parameters())
                print('Model restarted after ' +
                      str(num_steps_list[-1]) + ' steps')
        if i % args.print_freq == 0 and args.verbose is True:
            print('Epoch: [{0}/{1}][{2}/{3}]\t'
                  'LR: {LR:.6f}\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t'
                  'MAvg 1-err {current_ma:.4f}'.format(
                      epoch, args.epochs, i, len(train_loader), LR=1, batch_time=batch_time,
                      data_time=data_time, loss=losses, top1=top1, current_ma=current_ma))
    print('* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}'.format(
        epoch, args.epochs, top1=top1, loss=losses))
    return top1.avg, losses.avg, distill_data, model_losses.avg, \
        ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min, \
        model, optimizer
def train ( train_loader , model , distill_data , distill_labels , criterion , data_opt , epoch , optimizer , ma_list , ma_sum , lowest_ma_sum , current_num_steps , num_steps_list , num_steps_from_min , normalize ) : batch_time = AverageMeter ( ) data_time = AverageMeter ( ) losses = AverageMeter ( ) model_losses = AverageMeter ( ) top1 = AverageMeter ( ) model . train ( ) end = time . time ( ) for i , ( input_ , target ) in enumerate ( train_loader ) : data_time . update ( time . time ( ) - end ) input_ = input_ . to ( device = device ) target = target . to ( device = device ) perm = torch . randperm ( distill_data . size ( 0 ) ) idx = perm [ : args . inner_batch_size ] fi_i = torch . stack ( [ normalize ( image ) for image in distill_data [ idx ] . cpu ( ) ] ) . to ( device = device ) lb_i = distill_labels [ idx ] for weight in model . parameters ( ) : weight . fast = None fi_o = model ( fi_i ) loss = soft_cross_entropy ( fi_o , lb_i ) optimizer . zero_grad ( ) grad = torch . autograd . grad ( loss , model . parameters ( ) , create_graph = True ) for k , weight in enumerate ( model . parameters ( ) ) : weight . fast = weight - args . inner_lr * grad [ k ] logit = model ( input_ ) data_loss = criterion ( logit , target ) data_opt . zero_grad ( ) data_loss . backward ( retain_graph = False ) data_opt . step ( ) distill_data . data = torch . clamp ( distill_data . data , 0 , 1 ) fi_i = torch . stack ( [ normalize ( image ) for image in distill_data [ idx ] . cpu ( ) ] ) . to ( device = device ) lb_i = distill_labels [ idx ] for weight in model . parameters ( ) : weight . fast = None fi_o = model ( fi_i ) loss = soft_cross_entropy ( fi_o , lb_i ) optimizer . zero_grad ( ) loss . backward ( ) optimizer . step ( ) model_losses . update ( loss . item ( ) , input_ . size ( 0 ) ) err1 = compute_error_rate ( logit . data , target , topk = ( 1 , ) ) [ 0 ] losses . update ( data_loss . item ( ) , input_ . size ( 0 ) ) top1 . update ( err1 . item ( ) , input_ . size ( 0 ) ) batch_time . update ( time . time ( ) - end ) end = time . time ( ) if len ( ma_list ) < 50 : ma_list . append ( err1 . item ( ) ) ma_sum += err1 . item ( ) current_num_steps += 1 current_ma = ma_sum / len ( ma_list ) if current_num_steps == 50 : lowest_ma_sum = ma_sum num_steps_from_min = 0 else : ma_sum = ma_sum - ma_list [ 0 ] + err1 . item ( ) ma_list = ma_list [ 1 : ] + [ err1 . item ( ) ] current_num_steps += 1 current_ma = ma_sum / len ( ma_list ) if ma_sum < lowest_ma_sum : lowest_ma_sum = ma_sum num_steps_from_min = 0 elif num_steps_from_min < 50 : num_steps_from_min += 1 else : num_steps_list . append ( current_num_steps - num_steps_from_min - 1 ) ma_list = [ ] ma_sum = 0 lowest_ma_sum = 999999999 current_num_steps = 0 num_steps_from_min = 0 if args . target == "cifar10" : model = M . AlexCifarNetMeta ( args ) . to ( device = device ) else : model = M . LeNetMeta ( args ) . to ( device = device ) optimizer = torch . optim . Adam ( model . parameters ( ) ) print ( 'Model restarted after ' + str ( num_steps_list [ - 1 ] ) + ' steps' ) if i % args . print_freq == 0 and args . verbose is True : print ( 'Epoch: [{0}/{1}][{2}/{3}]\t' 'LR: {LR:.6f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Top 1-err {top1.val:.4f} ({top1.avg:.4f})\t' 'MAvg 1-err {current_ma:.4f}' . format ( epoch , args . epochs , i , len ( train_loader ) , LR = 1 , batch_time = batch_time , data_time = data_time , loss = losses , top1 = top1 , current_ma = current_ma ) ) print ( '* Epoch: [{0}/{1}]\t Top 1-err {top1.avg:.3f}  Train Loss {loss.avg:.3f}' . format ( epoch , args . epochs , top1 = top1 , loss = losses ) ) return top1 . avg , losses . avg , distill_data , model_losses . avg , ma_list , ma_sum , lowest_ma_sum , current_num_steps , num_steps_list , num_steps_from_min , model , optimizer
Do one epoch of training the synthetic images.
Do one epoch of training the synthetic images .
"""
    Do one epoch of training the synthetic images.

    Parameters ma_list, ma_sum, lowest_ma_sum, current_num_steps, num_steps_list, num_steps_from_min
    are used for keeping track of statistics for model resets across epochs.
    """ # switch to train mode # sample a minibatch of n_o target dataset examples x_t' with labels y_t' # measure data loading time # sample a minibatch of n_i examples from x~, y~: x~', y~' # normalize the synthetic images using the standard normalization # inner loop # create fast weights so that we can use second-order gradient # outer loop # update x~ <-- x~ - beta nabla_x~ L(f_w(f_theta(x_t)), y_t) # fast weights will be used # make sure the synthetic examples have valid values after the update # calculate the loss again for updating the features # measure error rate and record loss # it returns a list # measure elapsed time # update the moving average statistics # for image distillation, the values are fixed to 50 # do early stopping # restart all metrics # restart the model and the optimizer
train_loader
model
distill_data
distill_labels
criterion
data_opt
epoch
optimizer
ma_list
ma_sum
lowest_ma_sum
current_num_steps
num_steps_list
num_steps_from_min
normalize