def step(self, closure: Optional[Callable[([], torch.Tensor)]]=None, batch_size: Optional[int]=None, grad_scaler: Optional[GradScaler]=None):
    '\n        Update training progress after accumulating another local batch size. Depending on the configuration, this will\n        report progress to peers, run global or local optimizer step, average parameters or schedule background tasks.\n\n        :param closure: A closure that reevaluates the model and returns the loss.\n        :param batch_size: optional override for batch_size_per_step from init.\n        :param grad_scaler: if amp is enabled, this **must** be a hivemind-aware gradient scaler.\n        :note: this .step is different from normal pytorch optimizers in several key ways. See __init__ for details.\n        '
    if ((grad_scaler is not None) and (not isinstance(grad_scaler, GradScaler))):
        raise ValueError('hivemind.Optimizer requires a hivemind-aware gradient scaler (hivemind.GradScaler)')
    if ((self.batch_size_per_step is None) and (batch_size is None) and (not self.auxiliary)):
        raise ValueError('Please either set batch_size_per_step parameter at init or when calling .step')
    if (self.auxiliary and ((closure is not None) or (batch_size is not None) or (grad_scaler is not None))):
        raise ValueError('Auxiliary peers should not have batch size, run closures, or use grad_scaler')
    batch_size = (batch_size if (batch_size is not None) else self.batch_size_per_step)
    self.state_averager.step(apply_delayed_updates=True)
    loss = None
    if (closure is not None):
        with torch.enable_grad():
            loss = closure()
    if ((not self.auxiliary) and self._should_load_state_from_peers()):
        logger.log(self.status_loglevel, 'Peer is out of sync')
        self.load_state_from_peers()
        return loss
    if self.use_gradient_averaging:
        if (not self.auxiliary):
            grads_are_valid = self._check_and_accumulate_gradients(batch_size, grad_scaler)
            if (not grads_are_valid):
                return loss
        self._maybe_schedule_gradient_averaging()
        self._maybe_schedule_state_averaging()
    elif (not self.auxiliary):
        if (grad_scaler is not None):
            with grad_scaler.running_global_step():
                assert grad_scaler.unscale_(self)
        new_samples_accumulated = (self.tracker.local_progress.samples_accumulated + batch_size)
        self.tracker.report_local_progress(self.local_epoch, new_samples_accumulated)
        self._maybe_schedule_state_averaging()
        self.state_averager.step(increment_epoch=False, optimizer_step=True, delay_optimizer_step=self.delay_optimizer_step, grad_scaler=grad_scaler)
    if self.tracker.ready_to_update_epoch:
        self._update_global_epoch(grad_scaler)
    return loss
Update training progress after accumulating another local batch size. Depending on the configuration, this will
report progress to peers, run global or local optimizer step, average parameters or schedule background tasks.

:param closure: A closure that reevaluates the model and returns the loss.
:param batch_size: optional override for batch_size_per_step from init.
:param grad_scaler: if amp is enabled, this **must** be a hivemind-aware gradient scaler.
:note: this .step is different from normal pytorch optimizers in several key ways. See __init__ for details.
def step(self, closure: Optional[Callable[([], torch.Tensor)]]=None, batch_size: Optional[int]=None, grad_scaler: Optional[GradScaler]=None):
    '\n        Update training progress after accumulating another local batch size. Depending on the configuration, this will\n        report progress to peers, run global or local optimizer step, average parameters or schedule background tasks.\n\n        :param closure: A closure that reevaluates the model and returns the loss.\n        :param batch_size: optional override for batch_size_per_step from init.\n        :param grad_scaler: if amp is enabled, this **must** be a hivemind-aware gradient scaler.\n        :note: this .step is different from normal pytorch optimizers in several key ways. See __init__ for details.\n        '
    if ((grad_scaler is not None) and (not isinstance(grad_scaler, GradScaler))):
        raise ValueError('hivemind.Optimizer requires a hivemind-aware gradient scaler (hivemind.GradScaler)')
    if ((self.batch_size_per_step is None) and (batch_size is None) and (not self.auxiliary)):
        raise ValueError('Please either set batch_size_per_step parameter at init or when calling .step')
    if (self.auxiliary and ((closure is not None) or (batch_size is not None) or (grad_scaler is not None))):
        raise ValueError('Auxiliary peers should not have batch size, run closures, or use grad_scaler')
    batch_size = (batch_size if (batch_size is not None) else self.batch_size_per_step)
    self.state_averager.step(apply_delayed_updates=True)
    loss = None
    if (closure is not None):
        with torch.enable_grad():
            loss = closure()
    if ((not self.auxiliary) and self._should_load_state_from_peers()):
        logger.log(self.status_loglevel, 'Peer is out of sync')
        self.load_state_from_peers()
        return loss
    if self.use_gradient_averaging:
        if (not self.auxiliary):
            grads_are_valid = self._check_and_accumulate_gradients(batch_size, grad_scaler)
            if (not grads_are_valid):
                return loss
        self._maybe_schedule_gradient_averaging()
        self._maybe_schedule_state_averaging()
    elif (not self.auxiliary):
        if (grad_scaler is not None):
            with grad_scaler.running_global_step():
                assert grad_scaler.unscale_(self)
        new_samples_accumulated = (self.tracker.local_progress.samples_accumulated + batch_size)
        self.tracker.report_local_progress(self.local_epoch, new_samples_accumulated)
        self._maybe_schedule_state_averaging()
        self.state_averager.step(increment_epoch=False, optimizer_step=True, delay_optimizer_step=self.delay_optimizer_step, grad_scaler=grad_scaler)
    if self.tracker.ready_to_update_epoch:
        self._update_global_epoch(grad_scaler)
    return loss