Convert the rx/tx callback use counter from rte_smp_mb() and rte_smp_rmb() to C11 atomics.
The counter has a single writer per queue (MT_LOCKFREE tx is not handled, before or after), so use a relaxed load and store instead of a locked increment. Keep the seq_cst fence in inuse and wait for the counter/cb store-load pair. Use a release store in unuse paired with acquire loads in wait, so rte_bpf_destroy() is ordered after the last datapath use. Drop the extra fence in unload. With enable_stdatomic, x86 now does one locked instruction per burst instead of three. Signed-off-by: Stephen Hemminger <[email protected]> --- lib/bpf/bpf_pkt.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/bpf/bpf_pkt.c b/lib/bpf/bpf_pkt.c index f072fdaaed..f6acc69d25 100644 --- a/lib/bpf/bpf_pkt.c +++ b/lib/bpf/bpf_pkt.c @@ -80,9 +80,13 @@ static struct bpf_eth_cbh tx_cbh = { static __rte_always_inline void bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi) { - cbi->use++; - /* make sure no store/load reordering could happen */ - rte_smp_mb(); + uint32_t puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed); + + /* increment use count: does not need to be locked */ + rte_atomic_store_explicit(&cbi->use, puse + 1, rte_memory_order_relaxed); + + /* full barrier: count must be visible before cb is read */ + rte_atomic_thread_fence(rte_memory_order_seq_cst); } /* @@ -91,9 +95,10 @@ bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi) static __rte_always_inline void bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi) { - /* make sure all previous loads are completed */ - rte_smp_rmb(); - cbi->use++; + uint32_t puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed); + + /* release: pairs with the acquire bpf_eth_cbi_wait() */ + rte_atomic_store_explicit(&cbi->use, puse + 1, rte_memory_order_release); } /* @@ -104,15 +109,15 @@ bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi) { uint32_t puse; - /* make sure all previous loads and stores are completed */ - rte_smp_mb(); + /* full barrier: cleared cb must be visible before counter is read */ + rte_atomic_thread_fence(rte_memory_order_seq_cst); - puse = cbi->use; + puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_acquire); /* in use, busy wait till current RX/TX iteration is finished */ if ((puse & BPF_ETH_CBI_INUSE) != 0) { RTE_WAIT_UNTIL_MASKED((__rte_atomic uint32_t *)(uintptr_t)&cbi->use, - UINT32_MAX, !=, puse, rte_memory_order_relaxed); + UINT32_MAX, !=, puse, rte_memory_order_acquire); } } @@ -439,7 +444,6 @@ bpf_eth_cbi_unload(struct bpf_eth_cbi *bc) { /* mark this cbi as empty */ bc->cb = NULL; - rte_smp_mb(); /* make sure datapath doesn't use bpf anymore, then destroy bpf */ bpf_eth_cbi_wait(bc); -- 2.53.0
