> When fast releasing mbufs, the mbufs are not accessed, so do not prefetch > them. > This saves a mbuf load operation for each fast released TX mbuf. > > When fast release of mbufs is enabled for a TX queue, cache the mbuf > mempool pointer in the TX queue structure. > This saves one mbuf load operation for each burst of fast released TX > mbufs. > > The txep->mbuf pointer is not used after the mbuf has been freed, so do > not reset the pointer. > This saves a txep store operation for each TX mbuf freed. > > Signed-off-by: Morten Brørup <m...@smartsharesystems.com> > --- > drivers/net/intel/common/tx.h | 5 +++ > .../i40e/i40e_recycle_mbufs_vec_common.c | 4 +- > drivers/net/intel/i40e/i40e_rxtx.c | 39 ++++++++++--------- > 3 files changed, 28 insertions(+), 20 deletions(-) > > diff --git a/drivers/net/intel/common/tx.h b/drivers/net/intel/common/tx.h > index b0a68bae44..54c9b845f7 100644 > --- a/drivers/net/intel/common/tx.h > +++ b/drivers/net/intel/common/tx.h > @@ -62,6 +62,11 @@ struct ci_tx_queue { > uint16_t tx_next_dd; > uint16_t tx_next_rs; > uint64_t offloads; > + /* Mempool pointer for fast release of mbufs. > + * NULL if disabled, UINTPTR_MAX if enabled and not yet known. > + * Initialized at first use. > + */ > + struct rte_mempool *fast_free_mp; > uint64_t mbuf_errors; > rte_iova_t tx_ring_dma; /* TX ring DMA address */ > bool tx_deferred_start; /* don't start this queue in dev start */ > diff --git a/drivers/net/intel/i40e/i40e_recycle_mbufs_vec_common.c > b/drivers/net/intel/i40e/i40e_recycle_mbufs_vec_common.c > index 2875c578af..a46605cee9 100644 > --- a/drivers/net/intel/i40e/i40e_recycle_mbufs_vec_common.c > +++ b/drivers/net/intel/i40e/i40e_recycle_mbufs_vec_common.c > @@ -106,7 +106,9 @@ i40e_recycle_tx_mbufs_reuse_vec(void *tx_queue, > if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { > /* Avoid txq contains buffers from unexpected mempool. */ > if (unlikely(recycle_rxq_info->mp > - != txep[0].mbuf->pool)) > + != (likely(txq->fast_free_mp != (void > *)UINTPTR_MAX) ? > + txq->fast_free_mp : > + (txq->fast_free_mp = txep[0].mbuf->pool)))) > return 0; > > /* Directly put mbufs from Tx to Rx. */ > diff --git a/drivers/net/intel/i40e/i40e_rxtx.c > b/drivers/net/intel/i40e/i40e_rxtx.c > index c3ff2e05c3..679c1340b8 100644 > --- a/drivers/net/intel/i40e/i40e_rxtx.c > +++ b/drivers/net/intel/i40e/i40e_rxtx.c > @@ -1332,7 +1332,7 @@ static __rte_always_inline int > i40e_tx_free_bufs(struct ci_tx_queue *txq) > { > struct ci_tx_entry *txep; > - uint16_t tx_rs_thresh = txq->tx_rs_thresh; > + const uint16_t tx_rs_thresh = txq->tx_rs_thresh; > uint16_t i = 0, j = 0; > struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; > const uint16_t k = RTE_ALIGN_FLOOR(tx_rs_thresh, > RTE_I40E_TX_MAX_FREE_BUF_SZ); > @@ -1345,41 +1345,40 @@ i40e_tx_free_bufs(struct ci_tx_queue *txq) > > txep = &txq->sw_ring[txq->tx_next_dd - (tx_rs_thresh - 1)]; > > - for (i = 0; i < tx_rs_thresh; i++) > - rte_prefetch0((txep + i)->mbuf); > - > if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { > + struct rte_mempool * const fast_free_mp = > + likely(txq->fast_free_mp != (void > *)UINTPTR_MAX) ? > + txq->fast_free_mp : > + (txq->fast_free_mp = txep[0].mbuf->pool); > +
Nit idea. Acked-by: Konstantin Ananyev <konstantin.anan...@huawei.com> Just as a suggestion for further improvement: can we update (& check) txq->fast_free_mp not at tx_free_bufs() time, but when we fill txep[] and filling txd[] based on mbuf values? In theory it should allow to remove the check above. Also, again in theory, it opens opportunity (with some extra effort) to use similar optimization rte_mempool_put_bulk) even for cases when RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE is not set. > if (k) { > for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ) { > - for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; > ++i, ++txep) { > + for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; > ++i, ++txep) > free[i] = txep->mbuf; > - txep->mbuf = NULL; > - } > - rte_mempool_put_bulk(free[0]->pool, (void > **)free, > + rte_mempool_put_bulk(fast_free_mp, (void > **)free, > RTE_I40E_TX_MAX_FREE_BUF_SZ); > } > } > > if (m) { > - for (i = 0; i < m; ++i, ++txep) { > + for (i = 0; i < m; ++i, ++txep) > free[i] = txep->mbuf; > - txep->mbuf = NULL; > - } > - rte_mempool_put_bulk(free[0]->pool, (void **)free, m); > + rte_mempool_put_bulk(fast_free_mp, (void **)free, m); > } > } else { > - for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { > + for (i = 0; i < tx_rs_thresh; i++) > + rte_prefetch0((txep + i)->mbuf); > + > + for (i = 0; i < tx_rs_thresh; ++i, ++txep) > rte_pktmbuf_free_seg(txep->mbuf); > - txep->mbuf = NULL; > - } > } > > - txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); > - txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); > + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + tx_rs_thresh); > + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + tx_rs_thresh); > if (txq->tx_next_dd >= txq->nb_tx_desc) > - txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); > + txq->tx_next_dd = (uint16_t)(tx_rs_thresh - 1); > > - return txq->tx_rs_thresh; > + return tx_rs_thresh; > } > > /* Populate 4 descriptors with data from 4 mbufs */ > @@ -2546,6 +2545,8 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, > txq->reg_idx = reg_idx; > txq->port_id = dev->data->port_id; > txq->offloads = offloads; > + txq->fast_free_mp = offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE ? > + (void *)UINTPTR_MAX : NULL; > txq->i40e_vsi = vsi; > txq->tx_deferred_start = tx_conf->tx_deferred_start; > > -- > 2.43.0