From: Tariq Toukan <tar...@mellanox.com>
Use INDIRECT_CALL_2() helper to avoid the cost of the indirect call
when/if CONFIG_RETPOLINE=y.
Signed-off-by: Tariq Toukan <tar...@mellanox.com>
Reviewed-by: Maxim Mikityanskiy <maxi...@mellanox.com>
Signed-off-by: Saeed Mahameed <sae...@mellanox.com>
---
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 36 ++++++++++++-------
.../ethernet/mellanox/mlx5/core/en/xsk/tx.c | 22 ++++++++++--
2 files changed, 44 insertions(+), 14 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index c9d308e919655..3514208111555 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -34,6 +34,7 @@
#include <net/xdp_sock_drv.h>
#include "en/xdp.h"
#include "en/params.h"
+#include <linux/indirect_call_wrapper.h>
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
{
@@ -55,6 +56,15 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct
mlx5e_xsk_param *xsk)
return MLX5E_HW2SW_MTU(params, SKB_MAX_HEAD(hr));
}
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq
*sq,
+ struct
mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info
*xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data
*xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+
static inline bool
mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
struct mlx5e_dma_info *di, struct xdp_buff *xdp)
@@ -114,7 +124,8 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq
*rq,
xdpi.page.di = *di;
}
- return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0);
+ return INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0);
}
/* returns true if packet was consumed by xdp */
@@ -237,7 +248,7 @@ enum {
MLX5E_XDP_CHECK_START_MPWQE = 2,
};
-static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)
+INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct
mlx5e_xdpsq *sq)
{
if (unlikely(!sq->mpwqe.wqe)) {
const u16 stop_room =
mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
@@ -256,10 +267,9 @@ static int mlx5e_xmit_xdp_frame_check_mpwqe(struct
mlx5e_xdpsq *sq)
return MLX5E_XDP_CHECK_OK;
}
-static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
- struct mlx5e_xdp_xmit_data *xdptxd,
- struct mlx5e_xdp_info *xdpi,
- int check_result)
+INDIRECT_CALLABLE_SCOPE bool
+mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data
*xdptxd,
+ struct mlx5e_xdp_info *xdpi, int check_result)
{
struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
struct mlx5e_xdpsq_stats *stats = sq->stats;
@@ -293,7 +303,7 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq
*sq,
return true;
}
-static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
+INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
{
if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {
/* SQ is full, ring doorbell */
@@ -305,10 +315,9 @@ static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq
*sq)
return MLX5E_XDP_CHECK_OK;
}
-static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
- struct mlx5e_xdp_xmit_data *xdptxd,
- struct mlx5e_xdp_info *xdpi,
- int check_result)
+INDIRECT_CALLABLE_SCOPE bool
+mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data
*xdptxd,
+ struct mlx5e_xdp_info *xdpi, int check_result)
{
struct mlx5_wq_cyc *wq = &sq->wq;
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
@@ -506,6 +515,7 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct
xdp_frame **frames,
struct xdp_frame *xdpf = frames[i];
struct mlx5e_xdp_xmit_data xdptxd;
struct mlx5e_xdp_info xdpi;
+ bool ret;
xdptxd.data = xdpf->data;
xdptxd.len = xdpf->len;
@@ -522,7 +532,9 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct
xdp_frame **frames,
xdpi.frame.xdpf = xdpf;
xdpi.frame.dma_addr = xdptxd.dma_addr;
- if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0))) {
+ ret = INDIRECT_CALL_2(sq->xmit_xdp_frame,
mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi,
0);
+ if (unlikely(!ret)) {
dma_unmap_single(sq->pdev, xdptxd.dma_addr,
xdptxd.len, DMA_TO_DEVICE);
xdp_return_frame_rx_napi(xdpf);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
index e0b3c61af93ea..44f2d7bcf4bb2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
@@ -6,6 +6,7 @@
#include "en/xdp.h"
#include "en/params.h"
#include <net/xdp_sock_drv.h>
+#include <linux/indirect_call_wrapper.h>
int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
{
@@ -64,6 +65,17 @@ static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
sq->doorbell_cseg = &nopwqe->ctrl;
}
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq
*sq,
+ struct
mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info
*xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data
*xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct
mlx5e_xdpsq *sq));
+INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq
*sq));
+
bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
{
struct xdp_umem *umem = sq->umem;
@@ -75,8 +87,12 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int
budget)
xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;
for (; budget; budget--) {
- int check_result = sq->xmit_xdp_frame_check(sq);
+ int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check,
+
mlx5e_xmit_xdp_frame_check_mpwqe,
+ mlx5e_xmit_xdp_frame_check,
+ sq);
struct xdp_desc desc;
+ bool ret;
if (unlikely(check_result < 0)) {
work_done = false;
@@ -98,7 +114,9 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int
budget)
xsk_buff_raw_dma_sync_for_device(umem, xdptxd.dma_addr,
xdptxd.len);
- if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi,
check_result))) {
+ ret = INDIRECT_CALL_2(sq->xmit_xdp_frame,
mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi,
check_result);
+ if (unlikely(!ret)) {
if (sq->mpwqe.wqe)
mlx5e_xdp_mpwqe_complete(sq);
--
2.26.2
