dfo.py
def _prep_dfo(self):
        """
        Configures the DFO training process. 
        """

        self.theta = torch.nn.utils.parameters_to_vector(self.model.parameters())
        self.dim = len(self.theta)
        self.num_expl_directions = self.train_cfg['num_expl_directions']
        self.structured = self.train_cfg.get('structured', False)
        self.step_opt_type = self.train_cfg.get('step_opt_type', 'sgd')
        self.step_opt_params = self.train_cfg.get('step_opt_params', {'start': 0.001})

        if 'sigma' in self.model_start:            # if we're retraining or continuing from a DFO checkpoint, overwrite config                                       
            self.sigma = self.model_start['sigma']                                        
            self.step_opt = STEPSIZE_HELPER[self.model_start['step_opt_type']].load_state(self.model_start['step_opt_params'])
        else:
            self.sigma = self.train_cfg['sigma']['start']
            self.step_opt = STEPSIZE_HELPER[self.step_opt_type](self.dim, self.step_opt_params)

        if self.use_cuda:
            self.step_opt.cudafy()

        self.grad_estimator = GradEstimator(est_type=self.train_cfg.get('grad_estimator', 'antithetic'), 
                                            sigma=self.sigma, use_only_best=self.train_cfg.get('use_only_best', False),
                                            normalizer=self.train_cfg.get('normalizer', 'sigma'))

        self.logger.info('DFO configured.')
        
        # inform user about parallelization settings
        self.parallel_func_evals = self.train_cfg.get('parallel_func_evals', False)
        if self.num_workers > 1:
            self.logger.info("Using {} workers.".format(self.num_workers))
            if self.parallel_func_evals:
                self.logger.info("Parallelization type: parallel function evaluations")
            else: 
                self.logger.info("Parallelization type: parallel iterations")
        else:
            self.logger.info("Using 1 worker.")

        Configures the DFO training process. 
        
Configures the DFO training process.
Configures the DFO training process .
def _prep_dfo(self):
        self.theta = torch.nn.utils.parameters_to_vector(self.model.parameters())
        self.dim = len(self.theta)
        self.num_expl_directions = self.train_cfg['num_expl_directions']
        self.structured = self.train_cfg.get('structured', False)
        self.step_opt_type = self.train_cfg.get('step_opt_type', 'sgd')
        self.step_opt_params = self.train_cfg.get('step_opt_params', {'start': 0.001})
        if 'sigma' in self.model_start:            
            self.sigma = self.model_start['sigma']                                        
            self.step_opt = STEPSIZE_HELPER[self.model_start['step_opt_type']].load_state(self.model_start['step_opt_params'])
        else:
            self.sigma = self.train_cfg['sigma']['start']
            self.step_opt = STEPSIZE_HELPER[self.step_opt_type](self.dim, self.step_opt_params)
        if self.use_cuda:
            self.step_opt.cudafy()
        self.grad_estimator = GradEstimator(est_type=self.train_cfg.get('grad_estimator', 'antithetic'), 
                                            sigma=self.sigma, use_only_best=self.train_cfg.get('use_only_best', False),
                                            normalizer=self.train_cfg.get('normalizer', 'sigma'))
        self.logger.info('DFO configured.')
        self.parallel_func_evals = self.train_cfg.get('parallel_func_evals', False)
        if self.num_workers > 1:
            self.logger.info("Using {} workers.".format(self.num_workers))
            if self.parallel_func_evals:
                self.logger.info("Parallelization type: parallel function evaluations")
            else: 
                self.logger.info("Parallelization type: parallel iterations")
        else:
            self.logger.info("Using 1 worker.")
def _prep_dfo ( self ) : self . theta = torch . nn . utils . parameters_to_vector ( self . model . parameters ( ) ) self . dim = len ( self . theta ) self . num_expl_directions = self . train_cfg [ 'num_expl_directions' ] self . structured = self . train_cfg . get ( 'structured' , False ) self . step_opt_type = self . train_cfg . get ( 'step_opt_type' , 'sgd' ) self . step_opt_params = self . train_cfg . get ( 'step_opt_params' , { 'start' : 0.001 } ) if 'sigma' in self . model_start : self . sigma = self . model_start [ 'sigma' ] self . step_opt = STEPSIZE_HELPER [ self . model_start [ 'step_opt_type' ] ] . load_state ( self . model_start [ 'step_opt_params' ] ) else : self . sigma = self . train_cfg [ 'sigma' ] [ 'start' ] self . step_opt = STEPSIZE_HELPER [ self . step_opt_type ] ( self . dim , self . step_opt_params ) if self . use_cuda : self . step_opt . cudafy ( ) self . grad_estimator = GradEstimator ( est_type = self . train_cfg . get ( 'grad_estimator' , 'antithetic' ) , sigma = self . sigma , use_only_best = self . train_cfg . get ( 'use_only_best' , False ) , normalizer = self . train_cfg . get ( 'normalizer' , 'sigma' ) ) self . logger . info ( 'DFO configured.' ) self . parallel_func_evals = self . train_cfg . get ( 'parallel_func_evals' , False ) if self . num_workers > 1 : self . logger . info ( "Using {} workers." . format ( self . num_workers ) ) if self . parallel_func_evals : self . logger . info ( "Parallelization type: parallel function evaluations" ) else : self . logger . info ( "Parallelization type: parallel iterations" ) else : self . logger . info ( "Using 1 worker." )
Configures the DFO training process.
Configures the DFO training process .
"""
        Configures the DFO training process. 
        """ # if we're retraining or continuing from a DFO checkpoint, overwrite config                                        # inform user about parallelization settings
self