python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py
def sync_global_norm(self, block, ring_ids):
        """
        prune gradient_clip related ops for params that not belong to cur shard
        prune: square, reduce_sum, elementwise_mul
        keep: sum, sqrt, elementwise_max, elementwise_div
        """
        # FIXME(wangxi): mp should prune duplicated param_grads
        for idx, op in reversed(list(enumerate(block.ops))):
            if not self._is_gradient_clip_op(op):
                continue

            if op.type == "sum":
                sum_res = op.desc.output_arg_names()[0]
                for ring_id in ring_ids:
                    if ring_id == -1: continue

                    idx = idx + 1
                    block._insert_op_without_sync(
                        idx,
                        type='c_allreduce_sum',
                        inputs={'X': sum_res},
                        outputs={'Out': sum_res},
                        attrs={
                            'ring_id': ring_id,
                            'op_namescope': "/gradient_clip_model_parallelism",
                            'use_calc_stream': True,
                            OP_ROLE_KEY: OpRole.Optimize,
                        })
                return

        prune gradient_clip related ops for params that not belong to cur shard
        prune: square, reduce_sum, elementwise_mul
        keep: sum, sqrt, elementwise_max, elementwise_div
        
prune gradient_clip related ops for params that not belong to cur shard
prune: square, reduce_sum, elementwise_mul
keep: sum, sqrt, elementwise_max, elementwise_div
prune gradient_clip related ops for params that not belong to cur shard prune : square reduce_sum elementwise_mul keep : sum sqrt elementwise_max elementwise_div
def sync_global_norm(self, block, ring_ids):
        for idx, op in reversed(list(enumerate(block.ops))):
            if not self._is_gradient_clip_op(op):
                continue
            if op.type == "sum":
                sum_res = op.desc.output_arg_names()[0]
                for ring_id in ring_ids:
                    if ring_id == -1: continue
                    idx = idx + 1
                    block._insert_op_without_sync(
                        idx,
                        type='c_allreduce_sum',
                        inputs={'X': sum_res},
                        outputs={'Out': sum_res},
                        attrs={
                            'ring_id': ring_id,
                            'op_namescope': "/gradient_clip_model_parallelism",
                            'use_calc_stream': True,
                            OP_ROLE_KEY: OpRole.Optimize,
                        })
                return
def sync_global_norm ( self , block , ring_ids ) : for idx , op in reversed ( list ( enumerate ( block . ops ) ) ) : if not self . _is_gradient_clip_op ( op ) : continue if op . type == "sum" : sum_res = op . desc . output_arg_names ( ) [ 0 ] for ring_id in ring_ids : if ring_id == - 1 : continue idx = idx + 1 block . _insert_op_without_sync ( idx , type = 'c_allreduce_sum' , inputs = { 'X' : sum_res } , outputs = { 'Out' : sum_res } , attrs = { 'ring_id' : ring_id , 'op_namescope' : "/gradient_clip_model_parallelism" , 'use_calc_stream' : True , OP_ROLE_KEY : OpRole . Optimize , } ) return
prune gradient_clip related ops for params that not belong to cur shard
prune: square, reduce_sum, elementwise_mul
keep: sum, sqrt, elementwise_max, elementwise_div
prune gradient_clip related ops for params that not belong to cur shard prune : square reduce_sum elementwise_mul keep : sum sqrt elementwise_max elementwise_div
"""
        prune gradient_clip related ops for params that not belong to cur shard
        prune: square, reduce_sum, elementwise_mul
        keep: sum, sqrt, elementwise_max, elementwise_div
        """ # FIXME(wangxi): mp should prune duplicated param_grads
self
block
ring_ids