dfo.py
def train_and_validate(self):
        """
        Governs the actual training.

        :return list valid_rewards: The obtained validation rewards.
        :return list params_over_time: Contains every iteration's theta.
        :return list gradients: Contains the gradients computed in every iteration.
        :return list update_steps: Contains the update steps taken every iteration. 
        """

        # first, configure everything (order of calls is fixed!)
        self._prep_data()
        self._prep_model()
        self._prep_dfo()
        
        # set the validation frequency
        self.valid_freq = self.train_cfg['validation_freq']

        # keeps the highest validation score; used for saving checkpoints
        best_valid_score = 0.0

        # record some stuff
        params_over_time = [self.theta.clone()]
        valid_rewards = []
        gradients = []
        update_steps = []

        self.logger.info('Starting training...')
    
        for iteration in range(self.iterations + 1):

            if iteration % self.valid_freq == 0:
                # evaluate the validation set with the current model weights
                self.current_data = self.dev_data
                valid_reward = self.get_reward(self.theta)

                self.logger.info('Iteration: {}\t Reward valid. set: {}'.format(iteration, valid_reward))
                valid_rewards.append(valid_reward)

                # if a better reward was achieved, save a checkpoint
                if valid_reward > best_valid_score:
                    self.save_checkpoint(iteration)
                    best_valid_score = valid_reward

            for batch in iter(self.train_iter):
                
                # set to a class attribute so the reward function has direct access
                self.current_data = batch
                
                # generate the exploration directions
                expl_directions = None
                
                # if we're using multiple workers with each evaluating N expl. directions
                if self.num_workers > 1 and not self.parallel_func_evals:
                    
                    # generate enough expl. directions
                    expl_directions = self.get_expl_directions(self.num_expl_directions * self.num_workers)
                    
                    with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_workers) as executor: 
                        
                        # give every worker N exploration directions to evaluate                               
                        futures = {executor.submit(self.collect_rewards, subset): i // self.num_expl_directions 
                                                   for subset, i in subarray_generator(expl_directions, self.num_expl_directions)}                                          
                        rewards = [0.0] * self.num_workers              
                        for f in concurrent.futures.as_completed(futures):    
                            rewards[futures[f]] = f.result() 
                    
                    # stack the rewards into one tensor  
                    rewards = torch.stack(rewards)

                    # prepare for the gradient estimation (needs to be a 1D array)
                    rewards = torch.cat((rewards[:, :self.num_expl_directions].flatten(),
                                         rewards[:, self.num_expl_directions:].flatten()))

                else:   # if we're doing parallel function evaluations or no parallelization at all

                    expl_directions = self.get_expl_directions(self.num_expl_directions)
                    rewards = self.collect_rewards(expl_directions)

                # estimate the gradient
                grad = self.grad_estimator.estimate(expl_directions, rewards)

                # update theta
                update_step = self.step_opt.update(self.theta, -grad)
                
                # keep track
                gradients.append(grad)
                update_steps.append(update_step)
                params_over_time.append(self.theta.clone())

                break    # we only 'see' one batch per iteration

        return valid_rewards, params_over_time, gradients, update_steps

        Governs the actual training.

        :return list valid_rewards: The obtained validation rewards.
        :return list params_over_time: Contains every iteration's theta.
        :return list gradients: Contains the gradients computed in every iteration.
        :return list update_steps: Contains the update steps taken every iteration. 
        
Governs the actual training.
:return list valid_rewards: The obtained validation rewards.
:return list params_over_time: Contains every iteration's theta.
:return list gradients: Contains the gradients computed in every iteration.
:return list update_steps: Contains the update steps taken every iteration.
Governs the actual training . : return list valid_rewards : The obtained validation rewards . : return list params_over_time : Contains every iteration ' s theta . : return list gradients : Contains the gradients computed in every iteration . : return list update_steps : Contains the update steps taken every iteration .
def train_and_validate(self):
        self._prep_data()
        self._prep_model()
        self._prep_dfo()
        self.valid_freq = self.train_cfg['validation_freq']
        best_valid_score = 0.0
        params_over_time = [self.theta.clone()]
        valid_rewards = []
        gradients = []
        update_steps = []
        self.logger.info('Starting training...')
        for iteration in range(self.iterations + 1):
            if iteration % self.valid_freq == 0:
                self.current_data = self.dev_data
                valid_reward = self.get_reward(self.theta)
                self.logger.info('Iteration: {}\t Reward valid. set: {}'.format(iteration, valid_reward))
                valid_rewards.append(valid_reward)
                if valid_reward > best_valid_score:
                    self.save_checkpoint(iteration)
                    best_valid_score = valid_reward
            for batch in iter(self.train_iter):
                self.current_data = batch
                expl_directions = None
                if self.num_workers > 1 and not self.parallel_func_evals:
                    expl_directions = self.get_expl_directions(self.num_expl_directions * self.num_workers)
                    with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_workers) as executor: 
                        futures = {executor.submit(self.collect_rewards, subset): i // self.num_expl_directions 
                                                   for subset, i in subarray_generator(expl_directions, self.num_expl_directions)}                                          
                        rewards = [0.0] * self.num_workers              
                        for f in concurrent.futures.as_completed(futures):    
                            rewards[futures[f]] = f.result() 
                    rewards = torch.stack(rewards)
                    rewards = torch.cat((rewards[:, :self.num_expl_directions].flatten(),
                                         rewards[:, self.num_expl_directions:].flatten()))
                else:   
                    expl_directions = self.get_expl_directions(self.num_expl_directions)
                    rewards = self.collect_rewards(expl_directions)
                grad = self.grad_estimator.estimate(expl_directions, rewards)
                update_step = self.step_opt.update(self.theta, -grad)
                gradients.append(grad)
                update_steps.append(update_step)
                params_over_time.append(self.theta.clone())
                break    
        return valid_rewards, params_over_time, gradients, update_steps
def train_and_validate ( self ) : self . _prep_data ( ) self . _prep_model ( ) self . _prep_dfo ( ) self . valid_freq = self . train_cfg [ 'validation_freq' ] best_valid_score = 0.0 params_over_time = [ self . theta . clone ( ) ] valid_rewards = [ ] gradients = [ ] update_steps = [ ] self . logger . info ( 'Starting training...' ) for iteration in range ( self . iterations + 1 ) : if iteration % self . valid_freq == 0 : self . current_data = self . dev_data valid_reward = self . get_reward ( self . theta ) self . logger . info ( 'Iteration: {}\t Reward valid. set: {}' . format ( iteration , valid_reward ) ) valid_rewards . append ( valid_reward ) if valid_reward > best_valid_score : self . save_checkpoint ( iteration ) best_valid_score = valid_reward for batch in iter ( self . train_iter ) : self . current_data = batch expl_directions = None if self . num_workers > 1 and not self . parallel_func_evals : expl_directions = self . get_expl_directions ( self . num_expl_directions * self . num_workers ) with concurrent . futures . ThreadPoolExecutor ( max_workers = self . num_workers ) as executor : futures = { executor . submit ( self . collect_rewards , subset ) : i // self . num_expl_directions for subset , i in subarray_generator ( expl_directions , self . num_expl_directions ) } rewards = [ 0.0 ] * self . num_workers for f in concurrent . futures . as_completed ( futures ) : rewards [ futures [ f ] ] = f . result ( ) rewards = torch . stack ( rewards ) rewards = torch . cat ( ( rewards [ : , : self . num_expl_directions ] . flatten ( ) , rewards [ : , self . num_expl_directions : ] . flatten ( ) ) ) else : expl_directions = self . get_expl_directions ( self . num_expl_directions ) rewards = self . collect_rewards ( expl_directions ) grad = self . grad_estimator . estimate ( expl_directions , rewards ) update_step = self . step_opt . update ( self . theta , - grad ) gradients . append ( grad ) update_steps . append ( update_step ) params_over_time . append ( self . theta . clone ( ) ) break return valid_rewards , params_over_time , gradients , update_steps
Governs the actual training.
Governs the actual training .
"""
        Governs the actual training.

        :return list valid_rewards: The obtained validation rewards.
        :return list params_over_time: Contains every iteration's theta.
        :return list gradients: Contains the gradients computed in every iteration.
        :return list update_steps: Contains the update steps taken every iteration. 
        """ # first, configure everything (order of calls is fixed!) # set the validation frequency # keeps the highest validation score; used for saving checkpoints # record some stuff # evaluate the validation set with the current model weights # if a better reward was achieved, save a checkpoint # set to a class attribute so the reward function has direct access # generate the exploration directions # if we're using multiple workers with each evaluating N expl. directions # generate enough expl. directions # give every worker N exploration directions to evaluate                                # stack the rewards into one tensor   # prepare for the gradient estimation (needs to be a 1D array) # if we're doing parallel function evaluations or no parallelization at all # estimate the gradient # update theta # keep track # we only 'see' one batch per iteration
self