deepy/multigpu/worker.py
def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        """
        Train the model in multi-GPU environment.
        """
        from platoon.channel import Worker
        from platoon.param_sync import EASGD, ASGD
        server_port = self._port
        param_map = self.create_param_map()
        # Initialize the worker
        worker = Worker(control_port=server_port)
        if self.config.learning_rate:
            worker.send_req({'init_schedule': self._schedule_params})
        self.sync_hyperparams(worker.send_req('sync_hyperparams')['sync_hyperparams'])
        easgd_alpha = worker.send_req('get_easgd_alpha')
        if self._using_easgd:
            self.logger.info("using EASGD with alpha={}".format(easgd_alpha))
        else:
            self.logger.info("using ASGD rule")
        rule = EASGD(easgd_alpha) if self._using_easgd else ASGD()
        worker.init_shared_params(param_map.values(), param_sync_rule=rule)
        worker.send_req({
            "set_names": None,
            "training_names": self.training_names,
            "evaluation_names": self.evaluation_names
        })
        # Load all training batches, consume vast memory here
        self.logger.info("started process {}".format(os.getpid()))
        self.logger.info("(proc {}) load training data".format(os.getpid()))
        train_batches = list(train_set)
        network_callback = bool(self.network.training_callbacks)
        trainer_callback = bool(self._iter_controllers)
        # Start from valid, so the performance when a worked join can be known
        worker.copy_to_local()
        if valid_set:
            self._run_valid(self.epoch, valid_set, dry_run=True)
            self.fix_costs()
        worker.send_req({
            "valid_done": None,
            "valid_costs": self.last_run_costs,
            "auto_save": self.config.auto_save
        })
        worker.copy_to_local()
        # Begin the loop
        while True:
            resp = worker.send_req('next')
            if resp == 'stop':
                break
            elif resp == 'wait':
                time.sleep(1)
            elif resp == 'get_num_batches':
                worker.send_req({'get_num_batches_done': len(train_batches)})
            elif 'eval' in resp:
                self.best_cost = resp['best_valid_cost']
                worker.copy_to_local()
                valid_costs = None
                test_costs = None
                if valid_set:
                    self._run_valid(self.epoch, valid_set)
                    self.fix_costs()
                    valid_costs = self.last_run_costs
                if test_set:
                    self._run_test(self.epoch, test_set)
                    self.fix_costs()
                    test_costs = self.last_run_costs
                worker.send_req({
                    "eval_done": None,
                    "valid_costs": valid_costs,
                    "test_costs": test_costs,
                    "auto_save": self.config.auto_save
                })
            elif 'valid' in resp:
                self.best_cost = resp['best_valid_cost']
                worker.copy_to_local()
                if valid_set:
                    self._run_valid(self.epoch, valid_set, dry_run=True)
                    self.fix_costs()
                worker.send_req({
                    "valid_done": None,
                    "valid_costs": self.last_run_costs,
                    "auto_save": self.config.auto_save
                })
            elif 'train' in resp:
                batch_ids = resp['train']
                batch_costs = [[] for _ in self.training_names]
                for batch_id in batch_ids:
                    x = train_batches[batch_id]
                    cost_x = self.learn(*x)
                    for i, cost in enumerate(cost_x):
                        batch_costs[i].append(cost)
                    self.last_cost = cost_x[0]
                if network_callback:
                    self.network.training_callback()
                if trainer_callback:
                    for func in self._iter_controllers:
                        func(self)
                worker.sync_params(synchronous=True)
                worker.send_req({'train_done': None, 'costs': [float(np.mean(c)) for c in batch_costs]})
            elif 'sync_hyperparams' in resp:
                self.sync_hyperparams(resp['sync_hyperparams'])
        worker.close()
        return []
def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        """
        Train the model in multi-GPU environment.
        """
        from platoon.channel import Worker
        from platoon.param_sync import EASGD, ASGD
        server_port = self._port
        param_map = self.create_param_map()
        # Initialize the worker
        worker = Worker(control_port=server_port)
        if self.config.learning_rate:
            worker.send_req({'init_schedule': self._schedule_params})
        self.sync_hyperparams(worker.send_req('sync_hyperparams')['sync_hyperparams'])
        easgd_alpha = worker.send_req('get_easgd_alpha')
        if self._using_easgd:
            self.logger.info("using EASGD with alpha={}".format(easgd_alpha))
        else:
            self.logger.info("using ASGD rule")
        rule = EASGD(easgd_alpha) if self._using_easgd else ASGD()
        worker.init_shared_params(param_map.values(), param_sync_rule=rule)
        worker.send_req({
            "set_names": None,
            "training_names": self.training_names,
            "evaluation_names": self.evaluation_names
        })
        # Load all training batches, consume vast memory here
        self.logger.info("started process {}".format(os.getpid()))
        self.logger.info("(proc {}) load training data".format(os.getpid()))
        train_batches = list(train_set)
        network_callback = bool(self.network.training_callbacks)
        trainer_callback = bool(self._iter_controllers)
        # Start from valid, so the performance when a worked join can be known
        worker.copy_to_local()
        if valid_set:
            self._run_valid(self.epoch, valid_set, dry_run=True)
            self.fix_costs()
        worker.send_req({
            "valid_done": None,
            "valid_costs": self.last_run_costs,
            "auto_save": self.config.auto_save
        })
        worker.copy_to_local()
        # Begin the loop
        while True:
            resp = worker.send_req('next')
            if resp == 'stop':
                break
            elif resp == 'wait':
                time.sleep(1)
            elif resp == 'get_num_batches':
                worker.send_req({'get_num_batches_done': len(train_batches)})
            elif 'eval' in resp:
                self.best_cost = resp['best_valid_cost']
                worker.copy_to_local()
                valid_costs = None
                test_costs = None
                if valid_set:
                    self._run_valid(self.epoch, valid_set)
                    self.fix_costs()
                    valid_costs = self.last_run_costs
                if test_set:
                    self._run_test(self.epoch, test_set)
                    self.fix_costs()
                    test_costs = self.last_run_costs
                worker.send_req({
                    "eval_done": None,
                    "valid_costs": valid_costs,
                    "test_costs": test_costs,
                    "auto_save": self.config.auto_save
                })
            elif 'valid' in resp:
                self.best_cost = resp['best_valid_cost']
                worker.copy_to_local()
                if valid_set:
                    self._run_valid(self.epoch, valid_set, dry_run=True)
                    self.fix_costs()
                worker.send_req({
                    "valid_done": None,
                    "valid_costs": self.last_run_costs,
                    "auto_save": self.config.auto_save
                })
            elif 'train' in resp:
                batch_ids = resp['train']
                batch_costs = [[] for _ in self.training_names]
                for batch_id in batch_ids:
                    x = train_batches[batch_id]
                    cost_x = self.learn(*x)
                    for i, cost in enumerate(cost_x):
                        batch_costs[i].append(cost)
                    self.last_cost = cost_x[0]
                if network_callback:
                    self.network.training_callback()
                if trainer_callback:
                    for func in self._iter_controllers:
                        func(self)
                worker.sync_params(synchronous=True)
                worker.send_req({'train_done': None, 'costs': [float(np.mean(c)) for c in batch_costs]})
            elif 'sync_hyperparams' in resp:
                self.sync_hyperparams(resp['sync_hyperparams'])
        worker.close()
        return []
def train ( self , train_set , valid_set = None , test_set = None , train_size = None ) : from platoon . channel import Worker from platoon . param_sync import EASGD , ASGD server_port = self . _port param_map = self . create_param_map ( ) # Initialize the worker worker = Worker ( control_port = server_port ) if self . config . learning_rate : worker . send_req ( { 'init_schedule' : self . _schedule_params } ) self . sync_hyperparams ( worker . send_req ( 'sync_hyperparams' ) [ 'sync_hyperparams' ] ) easgd_alpha = worker . send_req ( 'get_easgd_alpha' ) if self . _using_easgd : self . logger . info ( "using EASGD with alpha={}" . format ( easgd_alpha ) ) else : self . logger . info ( "using ASGD rule" ) rule = EASGD ( easgd_alpha ) if self . _using_easgd else ASGD ( ) worker . init_shared_params ( param_map . values ( ) , param_sync_rule = rule ) worker . send_req ( { "set_names" : None , "training_names" : self . training_names , "evaluation_names" : self . evaluation_names } ) # Load all training batches, consume vast memory here self . logger . info ( "started process {}" . format ( os . getpid ( ) ) ) self . logger . info ( "(proc {}) load training data" . format ( os . getpid ( ) ) ) train_batches = list ( train_set ) network_callback = bool ( self . network . training_callbacks ) trainer_callback = bool ( self . _iter_controllers ) # Start from valid, so the performance when a worked join can be known worker . copy_to_local ( ) if valid_set : self . _run_valid ( self . epoch , valid_set , dry_run = True ) self . fix_costs ( ) worker . send_req ( { "valid_done" : None , "valid_costs" : self . last_run_costs , "auto_save" : self . config . auto_save } ) worker . copy_to_local ( ) # Begin the loop while True : resp = worker . send_req ( 'next' ) if resp == 'stop' : break elif resp == 'wait' : time . sleep ( 1 ) elif resp == 'get_num_batches' : worker . send_req ( { 'get_num_batches_done' : len ( train_batches ) } ) elif 'eval' in resp : self . best_cost = resp [ 'best_valid_cost' ] worker . copy_to_local ( ) valid_costs = None test_costs = None if valid_set : self . _run_valid ( self . epoch , valid_set ) self . fix_costs ( ) valid_costs = self . last_run_costs if test_set : self . _run_test ( self . epoch , test_set ) self . fix_costs ( ) test_costs = self . last_run_costs worker . send_req ( { "eval_done" : None , "valid_costs" : valid_costs , "test_costs" : test_costs , "auto_save" : self . config . auto_save } ) elif 'valid' in resp : self . best_cost = resp [ 'best_valid_cost' ] worker . copy_to_local ( ) if valid_set : self . _run_valid ( self . epoch , valid_set , dry_run = True ) self . fix_costs ( ) worker . send_req ( { "valid_done" : None , "valid_costs" : self . last_run_costs , "auto_save" : self . config . auto_save } ) elif 'train' in resp : batch_ids = resp [ 'train' ] batch_costs = [ [ ] for _ in self . training_names ] for batch_id in batch_ids : x = train_batches [ batch_id ] cost_x = self . learn ( * x ) for i , cost in enumerate ( cost_x ) : batch_costs [ i ] . append ( cost ) self . last_cost = cost_x [ 0 ] if network_callback : self . network . training_callback ( ) if trainer_callback : for func in self . _iter_controllers : func ( self ) worker . sync_params ( synchronous = True ) worker . send_req ( { 'train_done' : None , 'costs' : [ float ( np . mean ( c ) ) for c in batch_costs ] } ) elif 'sync_hyperparams' in resp : self . sync_hyperparams ( resp [ 'sync_hyperparams' ] ) worker . close ( ) return [ ]
Train the model in multi-GPU environment.
Train the model in multi - GPU environment .