deepspeed/runtime/engine.py
def step(self, lr_kwargs=None):
        r"""Execute the weight update step after forward and backward propagation
        on effective_train_batch.
        """
        if self.wall_clock_breakdown():
            self.timers('step_microstep').start()
            self.timers('step').start()

        assert self.optimizer is not None, "must provide optimizer during " \
                                           "init in order to use step"
        report_progress = self.global_rank == 0 if self.global_rank else True

        self._step_applied = False  # assume False, will flip to True

        # Update the model when we reach gradient accumulation boundaries
        if self.is_gradient_accumulation_boundary():
            self.gas_boundary_ctr += 1

            if self.eigenvalue_enabled() and (
                    self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution() ==
                    0) and self.quantizer.any_precision_switch():
                log_dist(f'computing eigenvalue...', ranks=[0])
                self.block_eigenvalue = self.eigenvalue.compute_eigenvalue(
                    self.module,
                    self.device,
                    self.optimizer.cur_scale)

            if self.progressive_layer_drop:
                self.progressive_layer_drop.update_state(self.global_steps)

            if self.eigenvalue_enabled(
            ) and not self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution(
            ) and self.quantizer.any_precision_switch():
                self._take_model_step(lr_kwargs, self.block_eigenvalue)
            else:
                self._take_model_step(lr_kwargs)

        self.tput_timer.stop(report_progress)

        # Log learning rate
        if self.tensorboard_enabled():
            if self.is_gradient_accumulation_boundary():
                if self.global_rank == 0:
                    self.summary_events = [(f'Train/Samples/lr',
                                            self.get_lr()[0],
                                            self.global_samples)]
                    for event in self.summary_events:  # write_summary_events
                        self.summary_writer.add_scalar(event[0], event[1], event[2])
                    if self.fp16_enabled() and hasattr(self.optimizer, 'cur_scale'):
                        self.summary_events.append((f'Train/Samples/loss_scale',
                                                    self.optimizer.cur_scale,
                                                    self.global_samples))

                    if self.eigenvalue_enabled(
                    ) and not self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution(
                    ):
                        ev_values = self.block_eigenvalue.values()
                        for i in range(len(ev_values)):
                            self.summary_writer.add_scalar(
                                f'Train/Eigenvalues/ModelBlockParam_{i}',
                                self.ev_values[i][0],
                                self.global_samples)
                            self.summary_writer.flush()

                    for event in self.summary_events:  # write_summary_events
                        self.summary_writer.add_scalar(event[0], event[1], event[2])
                    self.summary_writer.flush()

        if self.wall_clock_breakdown():
            self.timers('step').stop()
            self.timers('step_microstep').stop()
            timer_names = [
                'forward_microstep',
                'backward_microstep',
                'backward_inner_microstep',
                'backward_allreduce_microstep',
                'step_microstep'
            ]
            self.timers.log(names=timer_names,
                            reset=False,
                            memory_breakdown=self.memory_breakdown())

            # Log timing
            if self.is_gradient_accumulation_boundary():
                if self.tensorboard_enabled():
                    if self.global_rank == 0:
                        self.summary_events = [
                            (f'Train/Samples/elapsed_time_ms_forward',
                             self.timers('forward').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward',
                             self.timers('backward').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward_inner',
                             self.timers('backward_inner').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward_allreduce',
                             self.timers('backward_allreduce').elapsed(reset=False) *
                             1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_step',
                             self.timers('step').elapsed(reset=False) * 1000.0,
                             self.global_samples)
                        ]
                        for event in self.summary_events:  # write_summary_events
                            self.summary_writer.add_scalar(event[0], event[1], event[2])
                        self.summary_writer.flush()

            if self.wall_clock_breakdown():
                self.timers.log([
                    'forward',
                    'backward',
                    'backward_inner',
                    'backward_allreduce',
                    'step'
                ],
                                reset=False)

        self.micro_steps += 1
r"""Execute the weight update step after forward and backward propagation
        on effective_train_batch.
        
r"""Execute the weight update step after forward and backward propagation
on effective_train_batch.
r " " " Execute the weight update step after forward and backward propagation on effective_train_batch .
def step(self, lr_kwargs=None):
        if self.wall_clock_breakdown():
            self.timers('step_microstep').start()
            self.timers('step').start()
        assert self.optimizer is not None, "must provide optimizer during " \
                                           "init in order to use step"
        report_progress = self.global_rank == 0 if self.global_rank else True
        self._step_applied = False  
        if self.is_gradient_accumulation_boundary():
            self.gas_boundary_ctr += 1
            if self.eigenvalue_enabled() and (
                    self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution() ==
                    0) and self.quantizer.any_precision_switch():
                log_dist(f'computing eigenvalue...', ranks=[0])
                self.block_eigenvalue = self.eigenvalue.compute_eigenvalue(
                    self.module,
                    self.device,
                    self.optimizer.cur_scale)
            if self.progressive_layer_drop:
                self.progressive_layer_drop.update_state(self.global_steps)
            if self.eigenvalue_enabled(
            ) and not self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution(
            ) and self.quantizer.any_precision_switch():
                self._take_model_step(lr_kwargs, self.block_eigenvalue)
            else:
                self._take_model_step(lr_kwargs)
        self.tput_timer.stop(report_progress)
        if self.tensorboard_enabled():
            if self.is_gradient_accumulation_boundary():
                if self.global_rank == 0:
                    self.summary_events = [(f'Train/Samples/lr',
                                            self.get_lr()[0],
                                            self.global_samples)]
                    for event in self.summary_events:  
                        self.summary_writer.add_scalar(event[0], event[1], event[2])
                    if self.fp16_enabled() and hasattr(self.optimizer, 'cur_scale'):
                        self.summary_events.append((f'Train/Samples/loss_scale',
                                                    self.optimizer.cur_scale,
                                                    self.global_samples))
                    if self.eigenvalue_enabled(
                    ) and not self.gas_boundary_ctr % self.eigenvalue_gas_boundary_resolution(
                    ):
                        ev_values = self.block_eigenvalue.values()
                        for i in range(len(ev_values)):
                            self.summary_writer.add_scalar(
                                f'Train/Eigenvalues/ModelBlockParam_{i}',
                                self.ev_values[i][0],
                                self.global_samples)
                            self.summary_writer.flush()
                    for event in self.summary_events:  
                        self.summary_writer.add_scalar(event[0], event[1], event[2])
                    self.summary_writer.flush()
        if self.wall_clock_breakdown():
            self.timers('step').stop()
            self.timers('step_microstep').stop()
            timer_names = [
                'forward_microstep',
                'backward_microstep',
                'backward_inner_microstep',
                'backward_allreduce_microstep',
                'step_microstep'
            ]
            self.timers.log(names=timer_names,
                            reset=False,
                            memory_breakdown=self.memory_breakdown())
            if self.is_gradient_accumulation_boundary():
                if self.tensorboard_enabled():
                    if self.global_rank == 0:
                        self.summary_events = [
                            (f'Train/Samples/elapsed_time_ms_forward',
                             self.timers('forward').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward',
                             self.timers('backward').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward_inner',
                             self.timers('backward_inner').elapsed(reset=False) * 1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_backward_allreduce',
                             self.timers('backward_allreduce').elapsed(reset=False) *
                             1000.0,
                             self.global_samples),
                            (f'Train/Samples/elapsed_time_ms_step',
                             self.timers('step').elapsed(reset=False) * 1000.0,
                             self.global_samples)
                        ]
                        for event in self.summary_events:  
                            self.summary_writer.add_scalar(event[0], event[1], event[2])
                        self.summary_writer.flush()
            if self.wall_clock_breakdown():
                self.timers.log([
                    'forward',
                    'backward',
                    'backward_inner',
                    'backward_allreduce',
                    'step'
                ],
                                reset=False)
        self.micro_steps += 1
def step ( self , lr_kwargs = None ) : if self . wall_clock_breakdown ( ) : self . timers ( 'step_microstep' ) . start ( ) self . timers ( 'step' ) . start ( ) assert self . optimizer is not None , "must provide optimizer during " "init in order to use step" report_progress = self . global_rank == 0 if self . global_rank else True self . _step_applied = False if self . is_gradient_accumulation_boundary ( ) : self . gas_boundary_ctr += 1 if self . eigenvalue_enabled ( ) and ( self . gas_boundary_ctr % self . eigenvalue_gas_boundary_resolution ( ) == 0 ) and self . quantizer . any_precision_switch ( ) : log_dist ( f'computing eigenvalue...' , ranks = [ 0 ] ) self . block_eigenvalue = self . eigenvalue . compute_eigenvalue ( self . module , self . device , self . optimizer . cur_scale ) if self . progressive_layer_drop : self . progressive_layer_drop . update_state ( self . global_steps ) if self . eigenvalue_enabled ( ) and not self . gas_boundary_ctr % self . eigenvalue_gas_boundary_resolution ( ) and self . quantizer . any_precision_switch ( ) : self . _take_model_step ( lr_kwargs , self . block_eigenvalue ) else : self . _take_model_step ( lr_kwargs ) self . tput_timer . stop ( report_progress ) if self . tensorboard_enabled ( ) : if self . is_gradient_accumulation_boundary ( ) : if self . global_rank == 0 : self . summary_events = [ ( f'Train/Samples/lr' , self . get_lr ( ) [ 0 ] , self . global_samples ) ] for event in self . summary_events : self . summary_writer . add_scalar ( event [ 0 ] , event [ 1 ] , event [ 2 ] ) if self . fp16_enabled ( ) and hasattr ( self . optimizer , 'cur_scale' ) : self . summary_events . append ( ( f'Train/Samples/loss_scale' , self . optimizer . cur_scale , self . global_samples ) ) if self . eigenvalue_enabled ( ) and not self . gas_boundary_ctr % self . eigenvalue_gas_boundary_resolution ( ) : ev_values = self . block_eigenvalue . values ( ) for i in range ( len ( ev_values ) ) : self . summary_writer . add_scalar ( f'Train/Eigenvalues/ModelBlockParam_{i}' , self . ev_values [ i ] [ 0 ] , self . global_samples ) self . summary_writer . flush ( ) for event in self . summary_events : self . summary_writer . add_scalar ( event [ 0 ] , event [ 1 ] , event [ 2 ] ) self . summary_writer . flush ( ) if self . wall_clock_breakdown ( ) : self . timers ( 'step' ) . stop ( ) self . timers ( 'step_microstep' ) . stop ( ) timer_names = [ 'forward_microstep' , 'backward_microstep' , 'backward_inner_microstep' , 'backward_allreduce_microstep' , 'step_microstep' ] self . timers . log ( names = timer_names , reset = False , memory_breakdown = self . memory_breakdown ( ) ) if self . is_gradient_accumulation_boundary ( ) : if self . tensorboard_enabled ( ) : if self . global_rank == 0 : self . summary_events = [ ( f'Train/Samples/elapsed_time_ms_forward' , self . timers ( 'forward' ) . elapsed ( reset = False ) * 1000.0 , self . global_samples ) , ( f'Train/Samples/elapsed_time_ms_backward' , self . timers ( 'backward' ) . elapsed ( reset = False ) * 1000.0 , self . global_samples ) , ( f'Train/Samples/elapsed_time_ms_backward_inner' , self . timers ( 'backward_inner' ) . elapsed ( reset = False ) * 1000.0 , self . global_samples ) , ( f'Train/Samples/elapsed_time_ms_backward_allreduce' , self . timers ( 'backward_allreduce' ) . elapsed ( reset = False ) * 1000.0 , self . global_samples ) , ( f'Train/Samples/elapsed_time_ms_step' , self . timers ( 'step' ) . elapsed ( reset = False ) * 1000.0 , self . global_samples ) ] for event in self . summary_events : self . summary_writer . add_scalar ( event [ 0 ] , event [ 1 ] , event [ 2 ] ) self . summary_writer . flush ( ) if self . wall_clock_breakdown ( ) : self . timers . log ( [ 'forward' , 'backward' , 'backward_inner' , 'backward_allreduce' , 'step' ] , reset = False ) self . micro_steps += 1
r"""Execute the weight update step after forward and backward propagation
on effective_train_batch.
r " " " Execute the weight update step after forward and backward propagation on effective_train_batch .
r"""Execute the weight update step after forward and backward propagation
        on effective_train_batch.
        """ # assume False, will flip to True # Update the model when we reach gradient accumulation boundaries # Log learning rate # write_summary_events # write_summary_events # Log timing # write_summary_events
self
lr_kwargs