src/dpdk/drivers/net/mlx4/mlx4_rxtx.c
static inline volatile struct mlx4_wqe_ctrl_seg *
mlx4_tx_burst_fill_tso_dsegs(struct rte_mbuf *buf,
			     struct txq *txq,
			     struct tso_info *tinfo,
			     volatile struct mlx4_wqe_data_seg *dseg,
			     volatile struct mlx4_wqe_ctrl_seg *ctrl)
{
	uint32_t lkey;
	int nb_segs = buf->nb_segs;
	int nb_segs_txbb;
	struct mlx4_sq *sq = &txq->msq;
	struct rte_mbuf *sbuf = buf;
	struct pv *pv = tinfo->pv;
	int *pv_counter = &tinfo->pv_counter;
	volatile struct mlx4_wqe_ctrl_seg *ctrl_next =
			(volatile struct mlx4_wqe_ctrl_seg *)
				((volatile uint8_t *)ctrl + tinfo->wqe_size);
	uint16_t data_len = sbuf->data_len - tinfo->tso_header_size;
	uintptr_t data_addr = rte_pktmbuf_mtod_offset(sbuf, uintptr_t,
						      tinfo->tso_header_size);

	do {
		/* how many dseg entries do we have in the current TXBB ? */
		nb_segs_txbb = (MLX4_TXBB_SIZE -
				((uintptr_t)dseg & (MLX4_TXBB_SIZE - 1))) >>
			       MLX4_SEG_SHIFT;
		switch (nb_segs_txbb) {
#ifdef RTE_LIBRTE_MLX4_DEBUG
		default:
			/* Should never happen. */
			rte_panic("%p: Invalid number of SGEs(%d) for a TXBB",
			(void *)txq, nb_segs_txbb);
			/* rte_panic never returns. */
			break;
#endif /* RTE_LIBRTE_MLX4_DEBUG */
		case 4:
			/* Memory region key for this memory pool. */
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			dseg->addr = rte_cpu_to_be_64(data_addr);
			dseg->lkey = lkey;
			/*
			 * This data segment starts at the beginning of a new
			 * TXBB, so we need to postpone its byte_count writing
			 * for later.
			 */
			pv[*pv_counter].dseg = dseg;
			/*
			 * Zero length segment is treated as inline segment
			 * with zero data.
			 */
			pv[(*pv_counter)++].val =
				rte_cpu_to_be_32(data_len ?
						 data_len :
						 0x80000000);
			if (--nb_segs == 0)
				return ctrl_next;
			/* Prepare next buf info */
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
			/* fallthrough */
		case 3:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			/* Prepare next buf info */
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
			/* fallthrough */
		case 2:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			/* Prepare next buf info */
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
			/* fallthrough */
		case 1:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			/* Prepare next buf info */
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
			/* fallthrough */
		}
		/* Wrap dseg if it points at the end of the queue. */
		if ((volatile uint8_t *)dseg >= sq->eob)
			dseg = (volatile struct mlx4_wqe_data_seg *)
					((volatile uint8_t *)dseg - sq->size);
	} while (true);
err:
	return NULL;
}
/**
 * Fill the TSO WQE data segments with info on buffers to transmit .
 *
 * @param buf
 *   Pointer to the first packet mbuf.
 * @param txq
 *   Pointer to Tx queue structure.
 * @param tinfo
 *   Pointer to TSO info to use.
 * @param dseg
 *   Pointer to the first data segment in the TSO WQE.
 * @param ctrl
 *   Pointer to the control segment in the TSO WQE.
 *
 * @return
 *   0 on success, negative value upon error.
 */
Fill the TSO WQE data segments with info on buffers to transmit .
@param buf
Pointer to the first packet mbuf.
@param txq
Pointer to Tx queue structure.
@param tinfo
Pointer to TSO info to use.
@param dseg
Pointer to the first data segment in the TSO WQE.
@param ctrl
Pointer to the control segment in the TSO WQE.

@return
0 on success, negative value upon error.
Fill the TSO WQE data segments with info on buffers to transmit . @param buf Pointer to the first packet mbuf . @param txq Pointer to Tx queue structure . @param tinfo Pointer to TSO info to use . @param dseg Pointer to the first data segment in the TSO WQE . @param ctrl Pointer to the control segment in the TSO WQE . @return 0 on success negative value upon error .
static inline volatile struct mlx4_wqe_ctrl_seg *
mlx4_tx_burst_fill_tso_dsegs(struct rte_mbuf *buf,
			     struct txq *txq,
			     struct tso_info *tinfo,
			     volatile struct mlx4_wqe_data_seg *dseg,
			     volatile struct mlx4_wqe_ctrl_seg *ctrl)
{
	uint32_t lkey;
	int nb_segs = buf->nb_segs;
	int nb_segs_txbb;
	struct mlx4_sq *sq = &txq->msq;
	struct rte_mbuf *sbuf = buf;
	struct pv *pv = tinfo->pv;
	int *pv_counter = &tinfo->pv_counter;
	volatile struct mlx4_wqe_ctrl_seg *ctrl_next =
			(volatile struct mlx4_wqe_ctrl_seg *)
				((volatile uint8_t *)ctrl + tinfo->wqe_size);
	uint16_t data_len = sbuf->data_len - tinfo->tso_header_size;
	uintptr_t data_addr = rte_pktmbuf_mtod_offset(sbuf, uintptr_t,
						      tinfo->tso_header_size);
	do {
		nb_segs_txbb = (MLX4_TXBB_SIZE -
				((uintptr_t)dseg & (MLX4_TXBB_SIZE - 1))) >>
			       MLX4_SEG_SHIFT;
		switch (nb_segs_txbb) {
#ifdef RTE_LIBRTE_MLX4_DEBUG
		default:
			rte_panic("%p: Invalid number of SGEs(%d) for a TXBB",
			(void *)txq, nb_segs_txbb);
			break;
#endif 
		case 4:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			dseg->addr = rte_cpu_to_be_64(data_addr);
			dseg->lkey = lkey;
			pv[*pv_counter].dseg = dseg;
			pv[(*pv_counter)++].val =
				rte_cpu_to_be_32(data_len ?
						 data_len :
						 0x80000000);
			if (--nb_segs == 0)
				return ctrl_next;
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
		case 3:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
		case 2:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
		case 1:
			lkey = mlx4_tx_mb2mr(txq, sbuf);
			if (unlikely(lkey == (uint32_t)-1))
				goto err;
			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
					rte_cpu_to_be_32(data_len ?
							 data_len :
							 0x80000000));
			if (--nb_segs == 0)
				return ctrl_next;
			sbuf = sbuf->next;
			dseg++;
			data_len = sbuf->data_len;
			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
		}
		if ((volatile uint8_t *)dseg >= sq->eob)
			dseg = (volatile struct mlx4_wqe_data_seg *)
					((volatile uint8_t *)dseg - sq->size);
	} while (true);
err:
	return NULL;
}
static inline volatile struct mlx4_wqe_ctrl_seg * mlx4_tx_burst_fill_tso_dsegs ( struct rte_mbuf * buf , struct txq * txq , struct tso_info * tinfo , volatile struct mlx4_wqe_data_seg * dseg , volatile struct mlx4_wqe_ctrl_seg * ctrl ) { uint32_t lkey ; int nb_segs = buf -> nb_segs ; int nb_segs_txbb ; struct mlx4_sq * sq = & txq -> msq ; struct rte_mbuf * sbuf = buf ; struct pv * pv = tinfo -> pv ; int * pv_counter = & tinfo -> pv_counter ; volatile struct mlx4_wqe_ctrl_seg * ctrl_next = ( volatile struct mlx4_wqe_ctrl_seg * ) ( ( volatile uint8_t * ) ctrl + tinfo -> wqe_size ) ; uint16_t data_len = sbuf -> data_len - tinfo -> tso_header_size ; uintptr_t data_addr = rte_pktmbuf_mtod_offset ( sbuf , uintptr_t , tinfo -> tso_header_size ) ; do { nb_segs_txbb = ( MLX4_TXBB_SIZE - ( ( uintptr_t ) dseg & ( MLX4_TXBB_SIZE - 1 ) ) ) >> MLX4_SEG_SHIFT ; switch ( nb_segs_txbb ) { #ifdef RTE_LIBRTE_MLX4_DEBUG default : rte_panic ( " " , ( void * ) txq , nb_segs_txbb ) ; break ; #endif case 4 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) goto err ; dseg -> addr = rte_cpu_to_be_64 ( data_addr ) ; dseg -> lkey = lkey ; pv [ * pv_counter ] . dseg = dseg ; pv [ ( * pv_counter ) ++ ] . val = rte_cpu_to_be_32 ( data_len ? data_len : 0x80000000 ) ; if ( -- nb_segs == 0 ) return ctrl_next ; sbuf = sbuf -> next ; dseg ++ ; data_len = sbuf -> data_len ; data_addr = rte_pktmbuf_mtod ( sbuf , uintptr_t ) ; case 3 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) goto err ; mlx4_fill_tx_data_seg ( dseg , lkey , data_addr , rte_cpu_to_be_32 ( data_len ? data_len : 0x80000000 ) ) ; if ( -- nb_segs == 0 ) return ctrl_next ; sbuf = sbuf -> next ; dseg ++ ; data_len = sbuf -> data_len ; data_addr = rte_pktmbuf_mtod ( sbuf , uintptr_t ) ; case 2 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) goto err ; mlx4_fill_tx_data_seg ( dseg , lkey , data_addr , rte_cpu_to_be_32 ( data_len ? data_len : 0x80000000 ) ) ; if ( -- nb_segs == 0 ) return ctrl_next ; sbuf = sbuf -> next ; dseg ++ ; data_len = sbuf -> data_len ; data_addr = rte_pktmbuf_mtod ( sbuf , uintptr_t ) ; case 1 : lkey = mlx4_tx_mb2mr ( txq , sbuf ) ; if ( unlikely ( lkey == ( uint32_t ) -1 ) ) goto err ; mlx4_fill_tx_data_seg ( dseg , lkey , data_addr , rte_cpu_to_be_32 ( data_len ? data_len : 0x80000000 ) ) ; if ( -- nb_segs == 0 ) return ctrl_next ; sbuf = sbuf -> next ; dseg ++ ; data_len = sbuf -> data_len ; data_addr = rte_pktmbuf_mtod ( sbuf , uintptr_t ) ; } if ( ( volatile uint8_t * ) dseg >= sq -> eob ) dseg = ( volatile struct mlx4_wqe_data_seg * ) ( ( volatile uint8_t * ) dseg - sq -> size ) ; } while ( true ) ; err : return NULL ; }
Fill the TSO WQE data segments with info on buffers to transmit .
Fill the TSO WQE data segments with info on buffers to transmit .
/* how many dseg entries do we have in the current TXBB ? */ /* Should never happen. */ /* rte_panic never returns. */ /* RTE_LIBRTE_MLX4_DEBUG */ /* Memory region key for this memory pool. */ /*
			 * This data segment starts at the beginning of a new
			 * TXBB, so we need to postpone its byte_count writing
			 * for later.
			 */ /*
			 * Zero length segment is treated as inline segment
			 * with zero data.
			 */ /* Prepare next buf info */ /* fallthrough */ /* Prepare next buf info */ /* fallthrough */ /* Prepare next buf info */ /* fallthrough */ /* Prepare next buf info */ /* fallthrough */ /* Wrap dseg if it points at the end of the queue. */