xdp_zc_queues bitmap tracks if queue is setup as XSK
pool and xsk_port_id array tracks which port the XSK
queue is assigned to for zero copy.
Signed-off-by: Roger Quadros <[email protected]>
---
drivers/net/ethernet/ti/Makefile | 2 +-
drivers/net/ethernet/ti/am65-cpsw-nuss.c | 21 ++++--
drivers/net/ethernet/ti/am65-cpsw-nuss.h | 20 +++++
drivers/net/ethernet/ti/am65-cpsw-xdp.c | 122 +++++++++++++++++++++++++++++++
4 files changed, 156 insertions(+), 9 deletions(-)
diff --git a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile
index
93c0a4d0e33a6fb725ad61c3ec0eab87d2d3f61a..96585a28fc7d73f61b888e5d1587d5123875db31
100644
--- a/drivers/net/ethernet/ti/Makefile
+++ b/drivers/net/ethernet/ti/Makefile
@@ -29,7 +29,7 @@ keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o
netcp_xgbepcsr.o cpsw_ale.
obj-$(CONFIG_TI_K3_CPPI_DESC_POOL) += k3-cppi-desc-pool.o
obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o
-ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o
cpsw_ale.o
+ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o
cpsw_ale.o am65-cpsw-xdp.o
ti-am65-cpsw-nuss-$(CONFIG_TI_AM65_CPSW_QOS) += am65-cpsw-qos.o
ti-am65-cpsw-nuss-$(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV) += am65-cpsw-switchdev.o
obj-$(CONFIG_TI_K3_AM65_CPTS) += am65-cpts.o
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index
f9e2286efa29bbb7056fda1fc82c38b479aae8bd..46523be93df27710be77b288c36c1a0f66d8ca8d
100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -58,9 +58,6 @@
#define AM65_CPSW_MAX_PORTS 8
-#define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
-#define AM65_CPSW_MAX_PACKET_SIZE 2024
-
#define AM65_CPSW_REG_CTL 0x004
#define AM65_CPSW_REG_STAT_PORT_EN 0x014
#define AM65_CPSW_REG_PTYPE 0x018
@@ -505,7 +502,7 @@ static inline void am65_cpsw_put_page(struct
am65_cpsw_rx_flow *flow,
static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
-static void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id, bool retain_page_pool)
+void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id, bool
retain_page_pool)
{
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct am65_cpsw_rx_flow *flow;
@@ -554,7 +551,7 @@ static void am65_cpsw_destroy_rxqs(struct am65_cpsw_common
*common, bool retain_
k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
}
-static int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id)
+int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id)
{
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct page_pool_params pp_params = {
@@ -663,7 +660,7 @@ static int am65_cpsw_create_rxqs(struct am65_cpsw_common
*common)
return ret;
}
-static void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id)
+void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id)
{
struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
@@ -697,7 +694,7 @@ static void am65_cpsw_destroy_txqs(struct am65_cpsw_common *common)
am65_cpsw_destroy_txq(common, id);
}
-static int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id)
+int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id)
{
struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
int ret;
@@ -1327,7 +1324,7 @@ static int am65_cpsw_nuss_rx_packets(struct
am65_cpsw_rx_flow *flow,
dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len,
DMA_FROM_DEVICE);
k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
- if (port->xdp_prog) {
+ if (am65_cpsw_xdp_is_enabled(port)) {
xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]);
xdp_prepare_buff(&xdp, page_addr, AM65_CPSW_HEADROOM,
pkt_len, false);
@@ -1961,6 +1958,9 @@ static int am65_cpsw_ndo_bpf(struct net_device *ndev,
struct netdev_bpf *bpf)
switch (bpf->command) {
case XDP_SETUP_PROG:
return am65_cpsw_xdp_prog_setup(ndev, bpf->prog);
+ case XDP_SETUP_XSK_POOL:
+ return am65_cpsw_xsk_setup_pool(ndev, bpf->xsk.pool,
+ bpf->xsk.queue_id);
default:
return -EINVAL;
}
@@ -3553,7 +3553,12 @@ static int am65_cpsw_nuss_probe(struct platform_device
*pdev)
common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
if (!common)
return -ENOMEM;
+
common->dev = dev;
+ common->xdp_zc_queues = devm_bitmap_zalloc(dev, AM65_CPSW_MAX_QUEUES,
+ GFP_KERNEL);
+ if (!common->xdp_zc_queues)
+ return -ENOMEM;
of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
if (!of_id)
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h
b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
index
917c37e4e89bd933d3001f6c35a62db01cd8da4c..31789b5e5e1fc96be20cce17234d0e16cdcea796
100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
@@ -23,8 +23,14 @@ struct am65_cpts;
#define AM65_CPSW_MAX_QUEUES 8 /* both TX & RX */
+#define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
+#define AM65_CPSW_MAX_PACKET_SIZE 2024
+
#define AM65_CPSW_PORT_VLAN_REG_OFFSET 0x014
+#define AM65_CPSW_RX_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC |\
+ DMA_ATTR_WEAK_ORDERING)
+
struct am65_cpsw_slave_data {
bool mac_only;
struct cpsw_sl *mac_sl;
@@ -190,6 +196,9 @@ struct am65_cpsw_common {
unsigned char switch_id[MAX_PHYS_ITEM_ID_LEN];
/* only for suspend/resume context restore */
u32 *ale_context;
+ /* XDP Zero Copy */
+ unsigned long *xdp_zc_queues;
+ int xsk_port_id[AM65_CPSW_MAX_QUEUES];
};
struct am65_cpsw_ndev_priv {
@@ -228,4 +237,15 @@ int am65_cpsw_nuss_update_tx_rx_chns(struct
am65_cpsw_common *common,
bool am65_cpsw_port_dev_check(const struct net_device *dev);
+int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id);
+void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id, bool
retain_page_pool);
+int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id);
+void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id);
+int am65_cpsw_xsk_setup_pool(struct net_device *ndev,
+ struct xsk_buff_pool *pool, u16 qid);
+int am65_cpsw_xsk_wakeup(struct net_device *ndev, u32 qid, u32 flags);
+static inline bool am65_cpsw_xdp_is_enabled(struct am65_cpsw_port *port)
+{
+ return !!READ_ONCE(port->xdp_prog);
+}
#endif /* AM65_CPSW_NUSS_H_ */
diff --git a/drivers/net/ethernet/ti/am65-cpsw-xdp.c
b/drivers/net/ethernet/ti/am65-cpsw-xdp.c
new file mode 100644
index
0000000000000000000000000000000000000000..89f43f7c83db35dba96621bae930172e0fc85b6a
--- /dev/null
+++ b/drivers/net/ethernet/ti/am65-cpsw-xdp.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
+ *
+ * Copyright (C) 2025 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <net/xsk_buff_pool.h>
+#include <net/xdp_sock_drv.h>
+#include "am65-cpsw-nuss.h"
+
+static int am65_cpsw_xsk_pool_enable(struct am65_cpsw_port *port,
+ struct xsk_buff_pool *pool, u16 qid)
+{
+ struct am65_cpsw_common *common = port->common;
+ struct am65_cpsw_rx_chn *rx_chn;
+ bool need_update;
+ u32 frame_size;
+ int ret;
+
+ /*
+ * As queues are shared between ports we can no longer
+ * support the case where zero copy (XSK Pool) is enabled
+ * for the queue on one port but not for other ports.
+ *
+ * Current solution is to drop the packet if Zero copy
+ * is not enabled for that port + queue but enabled for
+ * some other port + same queue.
+ */
+ if (test_bit(qid, common->xdp_zc_queues))
+ return -EINVAL;
+
+ rx_chn = &common->rx_chns;
+ if (qid >= common->rx_ch_num_flows || qid >= common->tx_ch_num)
+ return -EINVAL;
+
+ frame_size = xsk_pool_get_rx_frame_size(pool);
+ if (frame_size < AM65_CPSW_MAX_PACKET_SIZE)
+ return -EOPNOTSUPP;
+
+ ret = xsk_pool_dma_map(pool, rx_chn->dma_dev, AM65_CPSW_RX_DMA_ATTR);
+ if (ret) {
+ netdev_err(port->ndev, "Failed to map xsk pool\n");
+ return ret;
+ }
+
+ need_update = common->usage_count &&
+ am65_cpsw_xdp_is_enabled(port);
+ if (need_update) {
+ am65_cpsw_destroy_rxq(common, qid, true);
+ am65_cpsw_destroy_txq(common, qid);
+ }
+
+ set_bit(qid, common->xdp_zc_queues);
+ common->xsk_port_id[qid] = port->port_id;
+ if (need_update) {
+ am65_cpsw_create_rxq(common, qid);
+ am65_cpsw_create_txq(common, qid);
+ }
+
+ return 0;
+}
+
+static int am65_cpsw_xsk_pool_disable(struct am65_cpsw_port *port,
+ struct xsk_buff_pool *pool, u16 qid)
+{
+ struct am65_cpsw_common *common = port->common;
+ bool need_update;
+
+ if (qid >= common->rx_ch_num_flows || qid >= common->tx_ch_num)
+ return -EINVAL;
+
+ if (!test_bit(qid, common->xdp_zc_queues))
+ return -EINVAL;
+
+ pool = xsk_get_pool_from_qid(port->ndev, qid);
+ if (!pool)
+ return -EINVAL;
+
+ need_update = common->usage_count && am65_cpsw_xdp_is_enabled(port);
+ if (need_update) {
+ am65_cpsw_destroy_rxq(common, qid, true);
+ am65_cpsw_destroy_txq(common, qid);
+ synchronize_rcu();
+ }
+
+ xsk_pool_dma_unmap(pool, AM65_CPSW_RX_DMA_ATTR);
+ clear_bit(qid, common->xdp_zc_queues);
+ common->xsk_port_id[qid] = -EINVAL;
+ if (need_update) {
+ am65_cpsw_create_rxq(common, qid);
+ am65_cpsw_create_txq(common, qid);
+ }
+
+ return 0;
+}
+
+int am65_cpsw_xsk_setup_pool(struct net_device *ndev,
+ struct xsk_buff_pool *pool, u16 qid)
+{
+ struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+
+ return pool ? am65_cpsw_xsk_pool_enable(port, pool, qid) :
+ am65_cpsw_xsk_pool_disable(port, pool, qid);
+}
+
+int am65_cpsw_xsk_wakeup(struct net_device *ndev, u32 qid, u32 flags)
+{
+ struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
+ struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+
+ if (!netif_running(ndev) || !netif_carrier_ok(ndev))
+ return -ENETDOWN;
+
+ if (!am65_cpsw_xdp_is_enabled(port))
+ return -EINVAL;
+
+ if (qid >= common->rx_ch_num_flows || qid >= common->tx_ch_num)
+ return -EINVAL;
+
+ return 0;
+}
