deepspeed/runtime/engine.py
def _get_zero_param_shapes(self):
        """Returns a dict of name to shape mapping, only for the flattened fp32 weights saved by the
        optimizer. the names are exactly as in state_dict. The order is absolutely important, since
        the saved data is just flattened data with no identifiers and requires reconstruction in the
        same order it was saved.

        We can't rely on self.module.named_parameters() to get the saved tensors, as some params
        will be missing and others unsaved and then it'd be impossible to reconstruct state_dict
        from the flattened weights.

        optimizer.fp16_groups seems to be the easiest to use as it's in all zeroX versions.
        """
        param_group_shapes = []
        cnt = 0
        numel = 0

        # zero2 started using a round_robin_fp16_groups which is a shuffled version of fp16_groups -
        # if we don't use it, we get parameters ordered incorrectly
        if hasattr(self.optimizer, "round_robin_fp16_groups"):
            fp16_groups = self.optimizer.round_robin_fp16_groups
        else:
            fp16_groups = self.optimizer.fp16_groups

        for fp16_group in fp16_groups:
            param_shapes = OrderedDict()
            for param in fp16_group:
                cnt += 1
                numel += param.ds_numel if hasattr(param, "ds_numel") else param.numel()
                shape = param.ds_shape if hasattr(param, "ds_shape") else param.shape
                if param not in self.param_names:
                    raise ValueError(f"failed to find optimizer param in named params")
                name = self.param_names[param]
                param_shapes[name] = shape

                # uncomment to debug zero_to_fp32.py problems
                # if self.global_rank == 0: print(f"saving param {name} {shape} (numel={shape.numel()})")
            param_group_shapes.append(param_shapes)
        # if self.global_rank == 0: print(f"Total saved {numel} numels in {cnt} params")

        return param_group_shapes
Returns a dict of name to shape mapping, only for the flattened fp32 weights saved by the
        optimizer. the names are exactly as in state_dict. The order is absolutely important, since
        the saved data is just flattened data with no identifiers and requires reconstruction in the
        same order it was saved.

        We can't rely on self.module.named_parameters() to get the saved tensors, as some params
        will be missing and others unsaved and then it'd be impossible to reconstruct state_dict
        from the flattened weights.

        optimizer.fp16_groups seems to be the easiest to use as it's in all zeroX versions.
        
Returns a dict of name to shape mapping, only for the flattened fp32 weights saved by the
optimizer. the names are exactly as in state_dict. The order is absolutely important, since
the saved data is just flattened data with no identifiers and requires reconstruction in the
same order it was saved.

We can't rely on self.module.named_parameters() to get the saved tensors, as some params
will be missing and others unsaved and then it'd be impossible to reconstruct state_dict
from the flattened weights.

optimizer.fp16_groups seems to be the easiest to use as it's in all zeroX versions.
Returns a dict of name to shape mapping only for the flattened fp32 weights saved by the optimizer . the names are exactly as in state_dict . The order is absolutely important since the saved data is just flattened data with no identifiers and requires reconstruction in the same order it was saved . We can ' t rely on self . module . named_parameters () to get the saved tensors as some params will be missing and others unsaved and then it ' d be impossible to reconstruct state_dict from the flattened weights . optimizer . fp16_groups seems to be the easiest to use as it ' s in all zeroX versions .
def _get_zero_param_shapes(self):
        param_group_shapes = []
        cnt = 0
        numel = 0
        if hasattr(self.optimizer, "round_robin_fp16_groups"):
            fp16_groups = self.optimizer.round_robin_fp16_groups
        else:
            fp16_groups = self.optimizer.fp16_groups
        for fp16_group in fp16_groups:
            param_shapes = OrderedDict()
            for param in fp16_group:
                cnt += 1
                numel += param.ds_numel if hasattr(param, "ds_numel") else param.numel()
                shape = param.ds_shape if hasattr(param, "ds_shape") else param.shape
                if param not in self.param_names:
                    raise ValueError(f"failed to find optimizer param in named params")
                name = self.param_names[param]
                param_shapes[name] = shape
            param_group_shapes.append(param_shapes)
        return param_group_shapes
def _get_zero_param_shapes ( self ) : param_group_shapes = [ ] cnt = 0 numel = 0 if hasattr ( self . optimizer , "round_robin_fp16_groups" ) : fp16_groups = self . optimizer . round_robin_fp16_groups else : fp16_groups = self . optimizer . fp16_groups for fp16_group in fp16_groups : param_shapes = OrderedDict ( ) for param in fp16_group : cnt += 1 numel += param . ds_numel if hasattr ( param , "ds_numel" ) else param . numel ( ) shape = param . ds_shape if hasattr ( param , "ds_shape" ) else param . shape if param not in self . param_names : raise ValueError ( f"failed to find optimizer param in named params" ) name = self . param_names [ param ] param_shapes [ name ] = shape param_group_shapes . append ( param_shapes ) return param_group_shapes
Returns a dict of name to shape mapping, only for the flattened fp32 weights saved by the
optimizer.
Returns a dict of name to shape mapping only for the flattened fp32 weights saved by the optimizer .
"""Returns a dict of name to shape mapping, only for the flattened fp32 weights saved by the
        optimizer. the names are exactly as in state_dict. The order is absolutely important, since
        the saved data is just flattened data with no identifiers and requires reconstruction in the
        same order it was saved.

        We can't rely on self.module.named_parameters() to get the saved tensors, as some params
        will be missing and others unsaved and then it'd be impossible to reconstruct state_dict
        from the flattened weights.

        optimizer.fp16_groups seems to be the easiest to use as it's in all zeroX versions.
        """ # zero2 started using a round_robin_fp16_groups which is a shuffled version of fp16_groups - # if we don't use it, we get parameters ordered incorrectly # uncomment to debug zero_to_fp32.py problems # if self.global_rank == 0: print(f"saving param {name} {shape} (numel={shape.numel()})") # if self.global_rank == 0: print(f"Total saved {numel} numels in {cnt} params")
self