src/dpdk/drivers/net/mlx4/mlx4_mr.c
static void
mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
{
	struct mlx4_priv *priv = dev->data->dev_private;
	const struct rte_memseg_list *msl;
	struct mlx4_mr *mr;
	int ms_n;
	int i;
	int rebuild = 0;

	DEBUG("port %u free callback: addr=%p, len=%zu",
	      dev->data->port_id, addr, len);
	msl = rte_mem_virt2memseg_list(addr);
	/* addr and len must be page-aligned. */
	MLX4_ASSERT((uintptr_t)addr ==
		    RTE_ALIGN((uintptr_t)addr, msl->page_sz));
	MLX4_ASSERT(len == RTE_ALIGN(len, msl->page_sz));
	ms_n = len / msl->page_sz;
	rte_rwlock_write_lock(&priv->mr.rwlock);
	/* Clear bits of freed memsegs from MR. */
	for (i = 0; i < ms_n; ++i) {
		const struct rte_memseg *ms;
		struct mlx4_mr_cache entry;
		uintptr_t start;
		int ms_idx;
		uint32_t pos;

		/* Find MR having this memseg. */
		start = (uintptr_t)addr + i * msl->page_sz;
		mr = mr_lookup_dev_list(dev, &entry, start);
		if (mr == NULL)
			continue;
		MLX4_ASSERT(mr->msl); /* Can't be external memory. */
		ms = rte_mem_virt2memseg((void *)start, msl);
		MLX4_ASSERT(ms != NULL);
		MLX4_ASSERT(msl->page_sz == ms->hugepage_sz);
		ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
		pos = ms_idx - mr->ms_base_idx;
		MLX4_ASSERT(rte_bitmap_get(mr->ms_bmp, pos));
		MLX4_ASSERT(pos < mr->ms_bmp_n);
		DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
		      dev->data->port_id, (void *)mr, pos, (void *)start);
		rte_bitmap_clear(mr->ms_bmp, pos);
		if (--mr->ms_n == 0) {
			LIST_REMOVE(mr, mr);
			LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
			DEBUG("port %u remove MR(%p) from list",
			      dev->data->port_id, (void *)mr);
		}
		/*
		 * MR is fragmented or will be freed. the global cache must be
		 * rebuilt.
		 */
		rebuild = 1;
	}
	if (rebuild) {
		mr_rebuild_dev_cache(dev);
		/*
		 * Flush local caches by propagating invalidation across cores.
		 * rte_smp_wmb() is enough to synchronize this event. If one of
		 * freed memsegs is seen by other core, that means the memseg
		 * has been allocated by allocator, which will come after this
		 * free call. Therefore, this store instruction (incrementing
		 * generation below) will be guaranteed to be seen by other core
		 * before the core sees the newly allocated memory.
		 */
		++priv->mr.dev_gen;
		DEBUG("broadcasting local cache flush, gen=%d",
		      priv->mr.dev_gen);
		rte_smp_wmb();
	}
	rte_rwlock_write_unlock(&priv->mr.rwlock);
#ifdef RTE_LIBRTE_MLX4_DEBUG
	if (rebuild)
		mlx4_mr_dump_dev(dev);
#endif
}
/**
 * Callback for memory free event. Iterate freed memsegs and check whether it
 * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
 * result, the MR would be fragmented. If it becomes empty, the MR will be freed
 * later by mlx4_mr_garbage_collect().
 *
 * The global cache must be rebuilt if there's any change and this event has to
 * be propagated to dataplane threads to flush the local caches.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param addr
 *   Address of freed memory.
 * @param len
 *   Size of freed memory.
 */
Callback for memory free event. Iterate freed memsegs and check whether it
belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
result, the MR would be fragmented. If it becomes empty, the MR will be freed
later by mlx4_mr_garbage_collect().

The global cache must be rebuilt if there's any change and this event has to
be propagated to dataplane threads to flush the local caches.

@param dev
Pointer to Ethernet device.
@param addr
Address of freed memory.
@param len
Size of freed memory.
Callback for memory free event . Iterate freed memsegs and check whether it belongs to an existing MR . If found clear the bit from bitmap of MR . As a result the MR would be fragmented . If it becomes empty the MR will be freed later by mlx4_mr_garbage_collect () . The global cache must be rebuilt if there ' s any change and this event has to be propagated to dataplane threads to flush the local caches . @param dev Pointer to Ethernet device . @param addr Address of freed memory . @param len Size of freed memory .
static void
mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
{
	struct mlx4_priv *priv = dev->data->dev_private;
	const struct rte_memseg_list *msl;
	struct mlx4_mr *mr;
	int ms_n;
	int i;
	int rebuild = 0;
	DEBUG("port %u free callback: addr=%p, len=%zu",
	      dev->data->port_id, addr, len);
	msl = rte_mem_virt2memseg_list(addr);
	MLX4_ASSERT((uintptr_t)addr ==
		    RTE_ALIGN((uintptr_t)addr, msl->page_sz));
	MLX4_ASSERT(len == RTE_ALIGN(len, msl->page_sz));
	ms_n = len / msl->page_sz;
	rte_rwlock_write_lock(&priv->mr.rwlock);
	for (i = 0; i < ms_n; ++i) {
		const struct rte_memseg *ms;
		struct mlx4_mr_cache entry;
		uintptr_t start;
		int ms_idx;
		uint32_t pos;
		start = (uintptr_t)addr + i * msl->page_sz;
		mr = mr_lookup_dev_list(dev, &entry, start);
		if (mr == NULL)
			continue;
		MLX4_ASSERT(mr->msl); 
		ms = rte_mem_virt2memseg((void *)start, msl);
		MLX4_ASSERT(ms != NULL);
		MLX4_ASSERT(msl->page_sz == ms->hugepage_sz);
		ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
		pos = ms_idx - mr->ms_base_idx;
		MLX4_ASSERT(rte_bitmap_get(mr->ms_bmp, pos));
		MLX4_ASSERT(pos < mr->ms_bmp_n);
		DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
		      dev->data->port_id, (void *)mr, pos, (void *)start);
		rte_bitmap_clear(mr->ms_bmp, pos);
		if (--mr->ms_n == 0) {
			LIST_REMOVE(mr, mr);
			LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
			DEBUG("port %u remove MR(%p) from list",
			      dev->data->port_id, (void *)mr);
		}
		rebuild = 1;
	}
	if (rebuild) {
		mr_rebuild_dev_cache(dev);
		++priv->mr.dev_gen;
		DEBUG("broadcasting local cache flush, gen=%d",
		      priv->mr.dev_gen);
		rte_smp_wmb();
	}
	rte_rwlock_write_unlock(&priv->mr.rwlock);
#ifdef RTE_LIBRTE_MLX4_DEBUG
	if (rebuild)
		mlx4_mr_dump_dev(dev);
#endif
}
static void mlx4_mr_mem_event_free_cb ( struct rte_eth_dev * dev , const void * addr , size_t len ) { struct mlx4_priv * priv = dev -> data -> dev_private ; const struct rte_memseg_list * msl ; struct mlx4_mr * mr ; int ms_n ; int i ; int rebuild = 0 ; DEBUG ( " " , dev -> data -> port_id , addr , len ) ; msl = rte_mem_virt2memseg_list ( addr ) ; MLX4_ASSERT ( ( uintptr_t ) addr == RTE_ALIGN ( ( uintptr_t ) addr , msl -> page_sz ) ) ; MLX4_ASSERT ( len == RTE_ALIGN ( len , msl -> page_sz ) ) ; ms_n = len / msl -> page_sz ; rte_rwlock_write_lock ( & priv -> mr . rwlock ) ; for ( i = 0 ; i < ms_n ; ++ i ) { const struct rte_memseg * ms ; struct mlx4_mr_cache entry ; uintptr_t start ; int ms_idx ; uint32_t pos ; start = ( uintptr_t ) addr + i * msl -> page_sz ; mr = mr_lookup_dev_list ( dev , & entry , start ) ; if ( mr == NULL ) continue ; MLX4_ASSERT ( mr -> msl ) ; ms = rte_mem_virt2memseg ( ( void * ) start , msl ) ; MLX4_ASSERT ( ms != NULL ) ; MLX4_ASSERT ( msl -> page_sz == ms -> hugepage_sz ) ; ms_idx = rte_fbarray_find_idx ( & msl -> memseg_arr , ms ) ; pos = ms_idx - mr -> ms_base_idx ; MLX4_ASSERT ( rte_bitmap_get ( mr -> ms_bmp , pos ) ) ; MLX4_ASSERT ( pos < mr -> ms_bmp_n ) ; DEBUG ( " " , dev -> data -> port_id , ( void * ) mr , pos , ( void * ) start ) ; rte_bitmap_clear ( mr -> ms_bmp , pos ) ; if ( -- mr -> ms_n == 0 ) { LIST_REMOVE ( mr , mr ) ; LIST_INSERT_HEAD ( & priv -> mr . mr_free_list , mr , mr ) ; DEBUG ( " " , dev -> data -> port_id , ( void * ) mr ) ; } rebuild = 1 ; } if ( rebuild ) { mr_rebuild_dev_cache ( dev ) ; ++ priv -> mr . dev_gen ; DEBUG ( " " , priv -> mr . dev_gen ) ; rte_smp_wmb ( ) ; } rte_rwlock_write_unlock ( & priv -> mr . rwlock ) ; #ifdef RTE_LIBRTE_MLX4_DEBUG if ( rebuild ) mlx4_mr_dump_dev ( dev ) ; #endif }
Callback for memory free event.
Callback for memory free event .
/* addr and len must be page-aligned. */ /* Clear bits of freed memsegs from MR. */ /* Find MR having this memseg. */ /* Can't be external memory. */ /*
		 * MR is fragmented or will be freed. the global cache must be
		 * rebuilt.
		 */ /*
		 * Flush local caches by propagating invalidation across cores.
		 * rte_smp_wmb() is enough to synchronize this event. If one of
		 * freed memsegs is seen by other core, that means the memseg
		 * has been allocated by allocator, which will come after this
		 * free call. Therefore, this store instruction (incrementing
		 * generation below) will be guaranteed to be seen by other core
		 * before the core sees the newly allocated memory.
		 */