models/Transformer/class_wrapper.py
def train_(self, train_loader, test_loader, save_model=False, epochs=500, optm='Adam', weight_decay=5e-4,
            lr=1e-3, lr_scheduler_name=None, lr_decay_rate=0.3, eval_step=10,
            stop_threshold=1e-7):
        """
        The major training function. This would start the training using information given in the flags
        :return: None
        """
        print("Starting training now")
        cuda = True if torch.cuda.is_available() else False
        if cuda:
            self.model.cuda()

        # Construct optimizer after the model moved to GPU
        self.optm = self.make_optimizer(optm, lr, weight_decay)
        if lr_scheduler_name is not None:
            self.lr_scheduler = self.make_lr_scheduler(self.optm, lr_scheduler_name, lr_decay_rate)

        for epoch in range(epochs):
            # Set to Training Mode
            train_loss = 0
            self.model.train()
            for j, (geometry, spectra) in enumerate(train_loader):
                if cuda:
                    geometry = geometry.cuda()                          # Put data onto GPU
                    spectra = spectra.cuda()                            # Put data onto GPU
                self.optm.zero_grad()                               # Zero the gradient first
                S_pred = self.model(geometry)
                loss = self.make_loss(logit=S_pred, labels=spectra)
                loss.backward()                                     # Calculate the backward gradients
                self.optm.step()                                    # Move one step the optimizer
                train_loss += loss.cpu().data.numpy()                                  # Aggregate the loss
                del S_pred, loss

            # Calculate the avg loss of training
            train_avg_loss = train_loss / (j + 1)
            # Recording the best training loss
            if train_avg_loss < self.best_training_loss:
                self.best_training_loss = train_avg_loss

            if epoch % eval_step == 0:                      # For eval steps, do the evaluations and tensor board
                # Record the training loss to the tensorboard
                self.log.add_scalar('Loss/total_train', train_avg_loss, epoch)

                # Set to Evaluation Mode
                self.model.eval()
                print("Doing Evaluation on the model now")
                test_loss = 0
                for j, (geometry, spectra) in enumerate(test_loader):  # Loop through the eval set
                    if cuda:
                        geometry = geometry.cuda()
                        spectra = spectra.cuda()

                    S_pred = self.model(geometry)
                    loss = self.make_loss(logit=S_pred, labels=spectra)
                    test_loss += loss.cpu().data.numpy()                                       # Aggregate the loss
                    del loss, S_pred

                # Record the testing loss to the tensorboard
                test_avg_loss = test_loss/ (j+1)
                self.log.add_scalar('Loss/total_test', test_avg_loss, epoch)

                print("This is Epoch %d, training loss %.5f, validation loss %.5f" \
                      % (epoch, train_avg_loss, test_avg_loss ))

                # Model improving, save the model down
                if test_avg_loss < self.best_validation_loss:
                    self.best_validation_loss = test_avg_loss
                    if save_model:
                        print("Saving the model down...")
                        self.save()

                    if self.best_validation_loss < stop_threshold:
                        print("Training finished EARLIER at epoch %d, reaching loss of %.5f" %\
                              (epoch, self.best_validation_loss))
                        break

            if lr_scheduler_name is not None:
                # Learning rate decay upon plateau
                self.lr_scheduler.step(train_avg_loss)

        The major training function. This would start the training using information given in the flags
        :return: None
        
The major training function. This would start the training using information given in the flags
The major training function . This would start the training using information given in the flags
def train_(self, train_loader, test_loader, save_model=False, epochs=500, optm='Adam', weight_decay=5e-4,
            lr=1e-3, lr_scheduler_name=None, lr_decay_rate=0.3, eval_step=10,
            stop_threshold=1e-7):
        print("Starting training now")
        cuda = True if torch.cuda.is_available() else False
        if cuda:
            self.model.cuda()
        self.optm = self.make_optimizer(optm, lr, weight_decay)
        if lr_scheduler_name is not None:
            self.lr_scheduler = self.make_lr_scheduler(self.optm, lr_scheduler_name, lr_decay_rate)
        for epoch in range(epochs):
            train_loss = 0
            self.model.train()
            for j, (geometry, spectra) in enumerate(train_loader):
                if cuda:
                    geometry = geometry.cuda()                          
                    spectra = spectra.cuda()                            
                self.optm.zero_grad()                               
                S_pred = self.model(geometry)
                loss = self.make_loss(logit=S_pred, labels=spectra)
                loss.backward()                                     
                self.optm.step()                                    
                train_loss += loss.cpu().data.numpy()                                  
                del S_pred, loss
            train_avg_loss = train_loss / (j + 1)
            if train_avg_loss < self.best_training_loss:
                self.best_training_loss = train_avg_loss
            if epoch % eval_step == 0:                      
                self.log.add_scalar('Loss/total_train', train_avg_loss, epoch)
                self.model.eval()
                print("Doing Evaluation on the model now")
                test_loss = 0
                for j, (geometry, spectra) in enumerate(test_loader):  
                    if cuda:
                        geometry = geometry.cuda()
                        spectra = spectra.cuda()
                    S_pred = self.model(geometry)
                    loss = self.make_loss(logit=S_pred, labels=spectra)
                    test_loss += loss.cpu().data.numpy()                                       
                    del loss, S_pred
                test_avg_loss = test_loss/ (j+1)
                self.log.add_scalar('Loss/total_test', test_avg_loss, epoch)
                print("This is Epoch %d, training loss %.5f, validation loss %.5f" \
                      % (epoch, train_avg_loss, test_avg_loss ))
                if test_avg_loss < self.best_validation_loss:
                    self.best_validation_loss = test_avg_loss
                    if save_model:
                        print("Saving the model down...")
                        self.save()
                    if self.best_validation_loss < stop_threshold:
                        print("Training finished EARLIER at epoch %d, reaching loss of %.5f" %\
                              (epoch, self.best_validation_loss))
                        break
            if lr_scheduler_name is not None:
                self.lr_scheduler.step(train_avg_loss)
def train_ ( self , train_loader , test_loader , save_model = False , epochs = 500 , optm = 'Adam' , weight_decay = 5e-4 , lr = 1e-3 , lr_scheduler_name = None , lr_decay_rate = 0.3 , eval_step = 10 , stop_threshold = 1e-7 ) : print ( "Starting training now" ) cuda = True if torch . cuda . is_available ( ) else False if cuda : self . model . cuda ( ) self . optm = self . make_optimizer ( optm , lr , weight_decay ) if lr_scheduler_name is not None : self . lr_scheduler = self . make_lr_scheduler ( self . optm , lr_scheduler_name , lr_decay_rate ) for epoch in range ( epochs ) : train_loss = 0 self . model . train ( ) for j , ( geometry , spectra ) in enumerate ( train_loader ) : if cuda : geometry = geometry . cuda ( ) spectra = spectra . cuda ( ) self . optm . zero_grad ( ) S_pred = self . model ( geometry ) loss = self . make_loss ( logit = S_pred , labels = spectra ) loss . backward ( ) self . optm . step ( ) train_loss += loss . cpu ( ) . data . numpy ( ) del S_pred , loss train_avg_loss = train_loss / ( j + 1 ) if train_avg_loss < self . best_training_loss : self . best_training_loss = train_avg_loss if epoch % eval_step == 0 : self . log . add_scalar ( 'Loss/total_train' , train_avg_loss , epoch ) self . model . eval ( ) print ( "Doing Evaluation on the model now" ) test_loss = 0 for j , ( geometry , spectra ) in enumerate ( test_loader ) : if cuda : geometry = geometry . cuda ( ) spectra = spectra . cuda ( ) S_pred = self . model ( geometry ) loss = self . make_loss ( logit = S_pred , labels = spectra ) test_loss += loss . cpu ( ) . data . numpy ( ) del loss , S_pred test_avg_loss = test_loss / ( j + 1 ) self . log . add_scalar ( 'Loss/total_test' , test_avg_loss , epoch ) print ( "This is Epoch %d, training loss %.5f, validation loss %.5f" % ( epoch , train_avg_loss , test_avg_loss ) ) if test_avg_loss < self . best_validation_loss : self . best_validation_loss = test_avg_loss if save_model : print ( "Saving the model down..." ) self . save ( ) if self . best_validation_loss < stop_threshold : print ( "Training finished EARLIER at epoch %d, reaching loss of %.5f" % ( epoch , self . best_validation_loss ) ) break if lr_scheduler_name is not None : self . lr_scheduler . step ( train_avg_loss )
The major training function.
The major training function .
"""
        The major training function. This would start the training using information given in the flags
        :return: None
        """ # Construct optimizer after the model moved to GPU
 # Set to Training Mode
 # Put data onto GPU
 # Put data onto GPU
 # Zero the gradient first
 # Calculate the backward gradients
 # Move one step the optimizer
 # Aggregate the loss
 # Calculate the avg loss of training
 # Recording the best training loss
 # For eval steps, do the evaluations and tensor board
 # Record the training loss to the tensorboard
 # Set to Evaluation Mode
 # Loop through the eval set
 # Aggregate the loss
 # Record the testing loss to the tensorboard
 # Model improving, save the model down
 # Learning rate decay upon plateau

None
self
train_loader
test_loader
save_model
epochs
optm
weight_decay
lr
lr_scheduler_name
lr_decay_rate
eval_step
stop_threshold