From: Shuanglin Wang <[email protected]>
Rx on Thor could stop delivering packets. Independent issues in the
doorbell and vectorized Rx paths combined to cause this:
- bnxt_db_epoch_write(), bnxt_db_mpc_write() and bnxt_db_mpc_cq()
always OR'd the epoch bit into the doorbell key, even on rings
whose db_epoch_mask is unset (every chip but P7). On those rings
the epoch bit lands in doorbell key space the firmware interprets
differently, corrupting the doorbell write. Only apply the epoch bit
when db_epoch_mask is set, and shift the epoch value as a 64 bit
quantity.
- The AVX2, SSE and NEON Rx wrappers split a request into chunks of
rx_free_thresh packets. A small rx_free_thresh could force chunks
narrower than one vector loop, so clamp the chunk size to at least
one vector loop width.
- bnxt_rxq_rearm() computes nb from rxrearm_nb & ~0x3, which is zero
whenever fewer than four mbufs need rearming. It then fell through
and rang the doorbell a second time with no new descriptors posted.
Return early when nb is zero.
Fixes: 30656a1cace8 ("net/bnxt: refactor epoch setting")
Cc: [email protected]
Signed-off-by: Shuanglin Wang <[email protected]>
Signed-off-by: Mohammad Shuab Siddique <[email protected]>
---
drivers/net/bnxt/bnxt_ring.h | 15 ++++++++++-----
drivers/net/bnxt/bnxt_rxtx_vec_avx2.c | 9 ++++++---
drivers/net/bnxt/bnxt_rxtx_vec_common.h | 2 ++
drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 3 ++-
drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 6 ++++--
5 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/drivers/net/bnxt/bnxt_ring.h b/drivers/net/bnxt/bnxt_ring.h
index d710d85371..d63ae8d8f8 100644
--- a/drivers/net/bnxt/bnxt_ring.h
+++ b/drivers/net/bnxt/bnxt_ring.h
@@ -116,7 +116,8 @@ static inline void bnxt_db_epoch_write(struct bnxt_db_info
*db, uint32_t idx, ui
if (db->db_64) {
uint64_t key_idx = db->db_key64 | db_idx;
- key_idx |= epoch << DBR_EPOCH_SFT;
+ if (db->db_epoch_mask)
+ key_idx |= ((uint64_t)epoch << DBR_EPOCH_SFT);
rte_compiler_barrier();
rte_write64_relaxed(key_idx, doorbell);
@@ -133,8 +134,10 @@ static inline void bnxt_db_mpc_write(struct bnxt_db_info
*db, uint32_t idx, uint
void *doorbell = db->doorbell;
if (likely(db->db_64)) {
- uint64_t key_idx = db->db_key64 | db_idx |
- (epoch << 24);
+ uint64_t key_idx = db->db_key64 | db_idx;
+
+ if (db->db_epoch_mask)
+ key_idx |= ((uint64_t)epoch << 24);
rte_write64_relaxed(key_idx, doorbell);
} else {
uint32_t key_idx = db->db_key32 | db_idx;
@@ -195,8 +198,10 @@ static inline void bnxt_db_mpc_cq(struct bnxt_cp_ring_info
*cpr)
uint32_t idx = DB_RING_IDX(&cpr->cp_db, cpr->cp_raw_cons);
if (likely(db->db_64)) {
- uint64_t key_idx = db->db_key64 | idx |
- (cpr->epoch << 24);
+ uint64_t key_idx = db->db_key64 | idx;
+
+ if (db->db_epoch_mask)
+ key_idx |= ((uint64_t)cpr->epoch << 24);
void *doorbell = db->doorbell;
rte_compiler_barrier();
diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
index 1236d3c2fe..a953a4a928 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
@@ -685,7 +685,8 @@ bnxt_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf
**rx_pkts,
uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t brst = rxq->rx_free_thresh;
+ uint32_t brst = (uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+ BNXT_RX_DESCS_PER_LOOP_VEC256);
uint16_t cnt = 0;
while (nb_pkts > brst) {
@@ -707,7 +708,8 @@ bnxt_crx_pkts_vec_avx2(void *rx_queue, struct rte_mbuf
**rx_pkts,
uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t brst = rxq->rx_free_thresh;
+ uint32_t brst = (uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+ BNXT_RX_DESCS_PER_LOOP_VEC256);
uint16_t cnt = 0;
while (nb_pkts > brst) {
@@ -1334,7 +1336,8 @@ bnxt_recv_pkts_vec_avx2_v3(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t expected_burst = rxq->rx_free_thresh;
+ uint32_t expected_burst =
(uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+
BNXT_RX_DESCS_PER_LOOP_VEC256);
uint16_t cnt = 0;
while (nb_pkts > expected_burst) {
diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h
b/drivers/net/bnxt/bnxt_rxtx_vec_common.h
index c75a8bd09f..bca18f936f 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h
@@ -67,6 +67,8 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct
bnxt_rx_ring_info *rxr)
*/
nb = RTE_MIN(rxq->rxrearm_nb & ~0x3,
rxq->nb_rx_desc - rxq->rxrearm_start);
+ if (unlikely(nb == 0))
+ return;
/* Allocate new mbufs into the software ring. */
if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) {
diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c
b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c
index 7aa153bc4e..d10e513f32 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c
@@ -318,7 +318,8 @@ uint16_t
bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t brst = rxq->rx_free_thresh;
+ uint32_t brst = (uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+ BNXT_RX_DESCS_PER_LOOP_VEC128);
uint16_t cnt = 0;
while (nb_pkts > brst) {
diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c
b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c
index 8dfb16a26d..e5ef48fb4c 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c
@@ -518,7 +518,8 @@ uint16_t
bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t brst = rxq->rx_free_thresh;
+ uint32_t brst = (uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+ BNXT_RX_DESCS_PER_LOOP_VEC128);
uint16_t cnt = 0;
while (nb_pkts > brst) {
@@ -540,7 +541,8 @@ uint16_t
bnxt_crx_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
struct bnxt_rx_queue *rxq = rx_queue;
- uint32_t brst = rxq->rx_free_thresh;
+ uint32_t brst = (uint16_t)RTE_MAX((uint32_t)rxq->rx_free_thresh,
+ BNXT_RX_DESCS_PER_LOOP_VEC128);
uint16_t cnt = 0;
while (nb_pkts > brst) {
--
2.47.3
