def METHOD_NAME(self):
    if self.world_size <= 1:
        return
    # Update the global model using local models from all GPUs
    # (Step-1) Calculate grad between previously synced model and
    # currrent local model
    if self.block_momentum != 0:
        self._calc_grad()
    # (Step-2) Average gradient from all GPUs
    self._avg_grad_from_all_gpus()
    # (Step-3) Calculate global momentum and update the global model
    if self.block_momentum != 0:
        self._update_global_model()
    # (Step-4) Average local optimizer params
    if self.average_sync:
        self.average_params()