src/dpdk/drivers/net/mlx4/mlx4_rxtx.c
static volatile struct mlx4_wqe_ctrl_seg *
mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
		   volatile struct mlx4_wqe_ctrl_seg *ctrl)
{
	struct pv *pv = (struct pv *)txq->bounce_buf;
	struct mlx4_sq *sq = &txq->msq;
	struct rte_mbuf *sbuf = buf;
	uint32_t lkey;
	int pv_counter = 0;
	int nb_segs = buf->nb_segs;
	uint32_t wqe_size;
	volatile struct mlx4_wqe_data_seg *dseg =
		(volatile struct mlx4_wqe_data_seg *)(ctrl + 1);

	ctrl->fence_size = 1 + nb_segs;
	wqe_size = RTE_ALIGN((uint32_t)(ctrl->fence_size << MLX4_SEG_SHIFT),
			     MLX4_TXBB_SIZE);
	/* Validate WQE size and WQE space in the send queue. */
	if (sq->remain_size < wqe_size ||
	    wqe_size > MLX4_MAX_WQE_SIZE)
		return NULL;
	/*
	 * Fill the data segments with buffer information.
	 * First WQE TXBB head segment is always control segment,
	 * so jump to tail TXBB data segments code for the first
	 * WQE data segments filling.
	 */
	goto txbb_tail_segs;
txbb_head_seg:
	/* Memory region key (big endian) for this memory pool. */
	lkey = mlx4_tx_mb2mr(txq, sbuf);
	if (unlikely(lkey == (uint32_t)-1)) {
		DEBUG("%p: unable to get MP <-> MR association",
		      (void *)txq);
		return NULL;
	}
	/* Handle WQE wraparound. */
	if (dseg >=
		(volatile struct mlx4_wqe_data_seg *)sq->eob)
		dseg = (volatile struct mlx4_wqe_data_seg *)
			sq->buf;
	dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
	dseg->lkey = lkey;
	/*
	 * This data segment starts at the beginning of a new
	 * TXBB, so we need to postpone its byte_count writing
	 * for later.
	 */
	pv[pv_counter].dseg = dseg;
	/*
	 * Zero length segment is treated as inline segment
	 * with zero data.
	 */
	pv[pv_counter++].val = rte_cpu_to_be_32(sbuf->data_len ?
						sbuf->data_len : 0x80000000);
	sbuf = sbuf->next;
	dseg++;
	nb_segs--;
txbb_tail_segs:
	/* Jump to default if there are more than two segments remaining. */
	switch (nb_segs) {
	default:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		sbuf = sbuf->next;
		dseg++;
		nb_segs--;
		/* fallthrough */
	case 2:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		sbuf = sbuf->next;
		dseg++;
		nb_segs--;
		/* fallthrough */
	case 1:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		nb_segs--;
		if (nb_segs) {
			sbuf = sbuf->next;
			dseg++;
			goto txbb_head_seg;
		}
		/* fallthrough */
	case 0:
		break;
	}
	/* Write the first DWORD of each TXBB save earlier. */
	if (pv_counter) {
		/* Need a barrier here before writing the byte_count. */
		rte_io_wmb();
		for (--pv_counter; pv_counter  >= 0; pv_counter--)
			pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
	}
	sq->remain_size -= wqe_size;
	/* Align next WQE address to the next TXBB. */
	return (volatile struct mlx4_wqe_ctrl_seg *)
		((volatile uint8_t *)ctrl + wqe_size);
}
/**
 * Write data segments of multi-segment packet.
 *
 * @param buf
 *   Pointer to the first packet mbuf.
 * @param txq
 *   Pointer to Tx queue structure.
 * @param ctrl
 *   Pointer to the WQE control segment.
 *
 * @return
 *   Pointer to the next WQE control segment on success, NULL otherwise.
 */
Write data segments of multi-segment packet.
@param buf
Pointer to the first packet mbuf.
@param txq
Pointer to Tx queue structure.
@param ctrl
Pointer to the WQE control segment.

@return
Pointer to the next WQE control segment on success, NULL otherwise.
Write data segments of multi - segment packet . @param buf Pointer to the first packet mbuf . @param txq Pointer to Tx queue structure . @param ctrl Pointer to the WQE control segment . @return Pointer to the next WQE control segment on success NULL otherwise .
static volatile struct mlx4_wqe_ctrl_seg *
mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
		   volatile struct mlx4_wqe_ctrl_seg *ctrl)
{
	struct pv *pv = (struct pv *)txq->bounce_buf;
	struct mlx4_sq *sq = &txq->msq;
	struct rte_mbuf *sbuf = buf;
	uint32_t lkey;
	int pv_counter = 0;
	int nb_segs = buf->nb_segs;
	uint32_t wqe_size;
	volatile struct mlx4_wqe_data_seg *dseg =
		(volatile struct mlx4_wqe_data_seg *)(ctrl + 1);
	ctrl->fence_size = 1 + nb_segs;
	wqe_size = RTE_ALIGN((uint32_t)(ctrl->fence_size << MLX4_SEG_SHIFT),
			     MLX4_TXBB_SIZE);
	if (sq->remain_size < wqe_size ||
	    wqe_size > MLX4_MAX_WQE_SIZE)
		return NULL;
	goto txbb_tail_segs;
txbb_head_seg:
	lkey = mlx4_tx_mb2mr(txq, sbuf);
	if (unlikely(lkey == (uint32_t)-1)) {
		DEBUG("%p: unable to get MP <-> MR association",
		      (void *)txq);
		return NULL;
	}
	if (dseg >=
		(volatile struct mlx4_wqe_data_seg *)sq->eob)
		dseg = (volatile struct mlx4_wqe_data_seg *)
			sq->buf;
	dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
	dseg->lkey = lkey;
	pv[pv_counter].dseg = dseg;
	pv[pv_counter++].val = rte_cpu_to_be_32(sbuf->data_len ?
						sbuf->data_len : 0x80000000);
	sbuf = sbuf->next;
	dseg++;
	nb_segs--;
txbb_tail_segs:
	switch (nb_segs) {
	default:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		sbuf = sbuf->next;
		dseg++;
		nb_segs--;
	case 2:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		sbuf = sbuf->next;
		dseg++;
		nb_segs--;
	case 1:
		lkey = mlx4_tx_mb2mr(txq, sbuf);
		if (unlikely(lkey == (uint32_t)-1)) {
			DEBUG("%p: unable to get MP <-> MR association",
			      (void *)txq);
			return NULL;
		}
		mlx4_fill_tx_data_seg(dseg, lkey,
				      rte_pktmbuf_mtod(sbuf, uintptr_t),
				      rte_cpu_to_be_32(sbuf->data_len ?
						       sbuf->data_len :
						       0x80000000));
		nb_segs--;
		if (nb_segs) {
			sbuf = sbuf->next;
			dseg++;
			goto txbb_head_seg;
		}
	case 0:
		break;
	}
	if (pv_counter) {
		rte_io_wmb();
		for (--pv_counter; pv_counter  >= 0; pv_counter--)
			pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
	}
	sq->remain_size -= wqe_size;
	return (volatile struct mlx4_wqe_ctrl_seg *)
		((volatile uint8_t *)ctrl + wqe_size);
}
static volatile struct mlx4_wqe_ctrl_seg * mlx4_tx_burst_segs ( struct rte_mbuf * buf , struct txq * txq , volatile struct mlx4_wqe_ctrl_seg * ctrl ) { struct pv * pv = ( struct pv * ) txq -> bounce_buf ; struct mlx4_sq * sq = & txq -> msq ; struct rte_mbuf * sbuf = buf ; uint32_t lkey ; int pv_counter = 0 ; int nb_segs = buf -> nb_segs ; uint32_t wqe_size ; volatile struct mlx4_wqe_data_seg * dseg = ( volatile struct mlx4_wqe_data_seg * ) ( ctrl + 1 ) ; ctrl -> fence_size = 1 + nb_segs ; wqe_size = RTE_ALIGN ( ( uint32_t ) ( ctrl -> fence_size << MLX4_SEG_SHIFT ) , MLX4_TXBB_SIZE ) ; if ( sq -> remain_size < wqe_size || wqe_size > MLX4_MAX_WQE_SIZE ) return NULL ; goto txbb_tail_segs ; txbb_head_seg : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) { DEBUG ( " " , ( void * ) txq ) ; return NULL ; } if ( dseg >= ( volatile struct mlx4_wqe_data_seg * ) sq -> eob ) dseg = ( volatile struct mlx4_wqe_data_seg * ) sq -> buf ; dseg -> addr = rte_cpu_to_be_64 ( rte_pktmbuf_mtod ( sbuf , uintptr_t ) ) ; dseg -> lkey = lkey ; pv [ pv_counter ] . dseg = dseg ; pv [ pv_counter ++ ] . val = rte_cpu_to_be_32 ( sbuf -> data_len ? sbuf -> data_len : 0x80000000 ) ; sbuf = sbuf -> next ; dseg ++ ; nb_segs -- ; txbb_tail_segs : switch ( nb_segs ) { default : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) { DEBUG ( " " , ( void * ) txq ) ; return NULL ; } mlx4_fill_tx_data_seg ( dseg , lkey , rte_pktmbuf_mtod ( sbuf , uintptr_t ) , rte_cpu_to_be_32 ( sbuf -> data_len ? sbuf -> data_len : 0x80000000 ) ) ; sbuf = sbuf -> next ; dseg ++ ; nb_segs -- ; case 2 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) { DEBUG ( " " , ( void * ) txq ) ; return NULL ; } mlx4_fill_tx_data_seg ( dseg , lkey , rte_pktmbuf_mtod ( sbuf , uintptr_t ) , rte_cpu_to_be_32 ( sbuf -> data_len ? sbuf -> data_len : 0x80000000 ) ) ; sbuf = sbuf -> next ; dseg ++ ; nb_segs -- ; case 1 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) { DEBUG ( " " , ( void * ) txq ) ; return NULL ; } mlx4_fill_tx_data_seg ( dseg , lkey , rte_pktmbuf_mtod ( sbuf , uintptr_t ) , rte_cpu_to_be_32 ( sbuf -> data_len ? sbuf -> data_len : 0x80000000 ) ) ; nb_segs -- ; if ( nb_segs ) { sbuf = sbuf -> next ; dseg ++ ; goto txbb_head_seg ; } case 0 : break ; } if ( pv_counter ) { rte_io_wmb ( ) ; for ( -- pv_counter ; pv_counter >= 0 ; pv_counter -- ) pv [ pv_counter ] . dseg -> byte_count = pv [ pv_counter ] . val ; } sq -> remain_size -= wqe_size ; return ( volatile struct mlx4_wqe_ctrl_seg * ) ( ( volatile uint8_t * ) ctrl + wqe_size ) ; }
Write data segments of multi-segment packet.
Write data segments of multi - segment packet .
/* Validate WQE size and WQE space in the send queue. */ /*
	 * Fill the data segments with buffer information.
	 * First WQE TXBB head segment is always control segment,
	 * so jump to tail TXBB data segments code for the first
	 * WQE data segments filling.
	 */ /* Memory region key (big endian) for this memory pool. */ /* Handle WQE wraparound. */ /*
	 * This data segment starts at the beginning of a new
	 * TXBB, so we need to postpone its byte_count writing
	 * for later.
	 */ /*
	 * Zero length segment is treated as inline segment
	 * with zero data.
	 */ /* Jump to default if there are more than two segments remaining. */ /* fallthrough */ /* fallthrough */ /* fallthrough */ /* Write the first DWORD of each TXBB save earlier. */ /* Need a barrier here before writing the byte_count. */ /* Align next WQE address to the next TXBB. */