The combined Tx burst path suffers from severe cache misses because the compact CQE and normal paths are interleaved. Split out a dedicated compact CQE entry point and select it directly from the runtime flag instead of the dynamic tx_ops indirection.
Signed-off-by: Jiacheng Ye <[email protected]> --- doc/guides/rel_notes/release_26_11.rst | 1 + drivers/net/hinic3/hinic3_ethdev.c | 31 +- drivers/net/hinic3/hinic3_ethdev.h | 1 - drivers/net/hinic3/hinic3_tx.c | 519 +++++++++++++++++++++---- drivers/net/hinic3/hinic3_tx.h | 35 +- 5 files changed, 455 insertions(+), 132 deletions(-) diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst index 53d0f76a32..8556fc5272 100644 --- a/doc/guides/rel_notes/release_26_11.rst +++ b/doc/guides/rel_notes/release_26_11.rst @@ -125,6 +125,7 @@ New Features * Added device parameters for runtime configuration. * Added a compact CQE receive data path. * Extended TCP segmentation offload (TSO) to 255 segments. + * Added a compact CQE transmit data path. * **Updated Intel iavf driver.** diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c index d3e95a361a..b64746664d 100644 --- a/drivers/net/hinic3/hinic3_ethdev.c +++ b/drivers/net/hinic3/hinic3_ethdev.c @@ -1392,6 +1392,7 @@ hinic3_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, (HINIC3_COS_NUM_MAX - 1) & nic_dev->cos_mask]; txq->tx_deferred_start = tx_conf->tx_deferred_start; + txq->tx_wqe_compact_task = HINIC3_SUPPORT_TX_WQE_COMPACT_TASK(nic_dev); ci_mz = hinic3_dma_zone_reserve(dev, "hinic3_sq_ci", qid, @@ -3715,14 +3716,6 @@ static const struct eth_dev_ops hinic3_pmd_vf_ops = { .eth_dev_priv_dump = hinic3_eth_dev_priv_dump, }; -static void hinic3_nic_tx_rx_ops_init(struct hinic3_nic_dev *nic_dev) -{ - if (HINIC3_SUPPORT_TX_WQE_COMPACT_TASK(nic_dev)) - nic_dev->tx_ops->nic_tx_set_wqe_offload = hinic3_tx_set_compact_task_offload; - else - nic_dev->tx_ops->nic_tx_set_wqe_offload = hinic3_tx_set_normal_task_offload; -} - /** * Initialize the network function, including hardware configuration, memory * allocation for data structures, MAC address setup, and interrupt enabling. @@ -3783,13 +3776,6 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) goto alloc_eth_addr_fail; } - nic_dev->tx_ops = rte_zmalloc("tx_ops", sizeof(struct hinic3_nic_tx_ops), 0); - if (!nic_dev->tx_ops) { - PMD_DRV_LOG(ERR, "Allocate tx_ops memory failed"); - err = -ENOMEM; - goto alloc_tx_ops_fail; - } - nic_dev->mc_list = rte_zmalloc("hinic3_mc", HINIC3_MAX_MC_MAC_ADDRS * sizeof(struct rte_ether_addr), 0); if (!nic_dev->mc_list) { @@ -3843,8 +3829,6 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) goto get_cap_fail; } - hinic3_nic_tx_rx_ops_init(nic_dev); - err = hinic3_init_sw_rxtxqs(nic_dev); if (err) { PMD_DRV_LOG(ERR, "Init sw rxqs or txqs failed, dev_name: %s", @@ -3934,10 +3918,6 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) nic_dev->mc_list = NULL; alloc_mc_list_fail: - rte_free(nic_dev->tx_ops); - nic_dev->tx_ops = NULL; - -alloc_tx_ops_fail: rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; @@ -4087,14 +4067,15 @@ hinic3_dev_init(struct rte_eth_dev *eth_dev) return err; } - if (nic_dev->config.rx_cqe_compact_en != HINIC3_RX_CQE_COMPACT_EN) + if (nic_dev->config.rx_cqe_compact_en != HINIC3_RX_CQE_COMPACT_EN) { eth_dev->rx_pkt_burst = hinic3_recv_pkts; - else + eth_dev->tx_pkt_burst = hinic3_xmit_pkts; + } else { eth_dev->rx_pkt_burst = hinic3_recv_pkts_compact_cqe; + eth_dev->tx_pkt_burst = hinic3_xmit_pkts_compact_cqe; + } } - eth_dev->tx_pkt_burst = hinic3_xmit_pkts; - return 0; } diff --git a/drivers/net/hinic3/hinic3_ethdev.h b/drivers/net/hinic3/hinic3_ethdev.h index 281953a8eb..92c7d44741 100644 --- a/drivers/net/hinic3/hinic3_ethdev.h +++ b/drivers/net/hinic3/hinic3_ethdev.h @@ -186,7 +186,6 @@ struct hinic3_nic_dev { struct hinic3_fdir_rule_filter_list filter_fdir_rule_list; uint8_t cos_map[HINIC3_COS_NUM_MAX]; struct hinic3_ptype_table *ptype_tbl; - struct hinic3_nic_tx_ops *tx_ops; enum nic_type nic_type; /**< NIC type. */ }; diff --git a/drivers/net/hinic3/hinic3_tx.c b/drivers/net/hinic3/hinic3_tx.c index c8a4170b87..b52be4c180 100644 --- a/drivers/net/hinic3/hinic3_tx.c +++ b/drivers/net/hinic3/hinic3_tx.c @@ -77,6 +77,30 @@ hinic3_get_and_update_sq_owner(struct hinic3_txq *sq, uint16_t curr_pi, uint16_t return owner; } +static void * +hinic3_get_sq_wqe(struct hinic3_txq *sq, + struct hinic3_wqe_info *wqe_info) +{ + uint16_t cur_pi = MASKED_QUEUE_IDX(sq, sq->prod_idx); + uint32_t end_pi; + + end_pi = cur_pi + wqe_info->wqebb_cnt; + sq->prod_idx += wqe_info->wqebb_cnt; + + wqe_info->owner = (uint8_t)(sq->owner); + wqe_info->pi = cur_pi; + wqe_info->wrapped = 0; + + if (unlikely(end_pi >= sq->q_depth)) { + sq->owner = !sq->owner; + + if (likely(end_pi > sq->q_depth)) + wqe_info->wrapped = (uint8_t)(sq->q_depth - cur_pi); + } + + return NIC_WQE_ADDR(sq, cur_pi); +} + static inline void hinic3_put_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info) { @@ -97,9 +121,9 @@ hinic3_put_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info) * Point to wqe_info of send queue(SQ). */ static void -hinic3_set_wqe_combo(struct hinic3_txq *sq, - struct hinic3_sq_wqe_combo *wqe_combo, - struct hinic3_wqe_info *wqe_info) +hinic3_set_wqe_combo_compact_cqe(struct hinic3_txq *sq, + struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) { uint16_t tmp_pi; @@ -124,6 +148,62 @@ hinic3_set_wqe_combo(struct hinic3_txq *sq, wqe_info->owner = hinic3_get_and_update_sq_owner(sq, wqe_info->pi, wqe_info->wqebb_cnt); } +/** + * Sets the WQE combination information in the transmit queue (SQ). + * + * @param[in] sq + * Point to send queue. + * @param[out] wqe_combo + * Point to wqe_combo of send queue(SQ). + * @param[in] wqe_info + * Point to wqe_info of send queue(SQ). + */ +static void +hinic3_set_wqe_combo(struct hinic3_txq *sq, + struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_sq_wqe *wqe, + struct hinic3_wqe_info *wqe_info) +{ + wqe_combo->hdr = &wqe->compact_wqe.wqe_desc; + + if (wqe_info->offload) { + if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) { + wqe_combo->task = (struct hinic3_sq_task *) + (void *)sq->sq_head_addr; + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(sq->sq_head_addr + sq->wqebb_size); + } else if (wqe_info->wrapped == HINIC3_TX_BD_DESC_WRAPPED) { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(sq->sq_head_addr); + } else { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = wqe->extend_wqe.buf_desc; + } + + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES; + return; + } + + if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) { + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(sq->sq_head_addr); + } else { + wqe_combo->bds_head = + (struct hinic3_sq_bufdesc *)(&wqe->extend_wqe.task); + } + + if (wqe_info->wqebb_cnt > 1) { + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_4BYTES; + /* This section used as vlan insert, needs to clear */ + wqe_combo->bds_head->rsvd = 0; + } else { + wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE; + } +} + int hinic3_start_all_sqs(struct rte_eth_dev *eth_dev) { @@ -343,33 +423,15 @@ hinic3_tx_offload_pkt_prepare(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *m return 0; } -void -hinic3_tx_set_normal_task_offload(struct hinic3_wqe_info *wqe_info, - struct hinic3_sq_wqe_combo *wqe_combo) +static inline void hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task, + uint16_t vlan_tag, uint8_t vlan_type) { - struct hinic3_sq_task *task = wqe_combo->task; - struct hinic3_offload_info *offload_info = &wqe_info->offload_info; - - task->pkt_info0 = 0; - task->pkt_info0 |= SQ_TASK_INFO0_SET(offload_info->inner_l4_en, INNER_L4_EN); - task->pkt_info0 |= SQ_TASK_INFO0_SET(offload_info->inner_l3_en, INNER_L3_EN); - task->pkt_info0 |= SQ_TASK_INFO0_SET(offload_info->encapsulation, TUNNEL_FLAG); - task->pkt_info0 |= SQ_TASK_INFO0_SET(offload_info->out_l3_en, OUT_L3_EN); - task->pkt_info0 |= SQ_TASK_INFO0_SET(offload_info->out_l4_en, OUT_L4_EN); - task->pkt_info0 = hinic3_hw_be32(task->pkt_info0); - - if (wqe_combo->task_type == SQ_WQE_TASKSECT_16BYTES) { - task->ip_identify = 0; - task->pkt_info2 = 0; - task->vlan_offload = 0; - task->vlan_offload = SQ_TASK_INFO3_SET(offload_info->vlan_tag, VLAN_TAG) | - SQ_TASK_INFO3_SET(offload_info->vlan_sel, VLAN_TYPE) | - SQ_TASK_INFO3_SET(offload_info->vlan_valid, VLAN_TAG_VALID); - task->vlan_offload = hinic3_hw_be32(task->vlan_offload); - } + task->vlan_offload = SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) | + SQ_TASK_INFO3_SET(vlan_type, VLAN_TYPE) | + SQ_TASK_INFO3_SET(1U, VLAN_TAG_VALID); } -void +static void hinic3_tx_set_compact_task_offload(struct hinic3_wqe_info *wqe_info, struct hinic3_sq_wqe_combo *wqe_combo) { @@ -390,10 +452,9 @@ hinic3_tx_set_compact_task_offload(struct hinic3_wqe_info *wqe_info, } static int -hinic3_set_tx_offload(struct hinic3_nic_dev *nic_dev, - struct rte_mbuf *mbuf, - struct hinic3_sq_wqe_combo *wqe_combo, - struct hinic3_wqe_info *wqe_info) +hinic3_set_tx_offload_compact_cqe(struct rte_mbuf *mbuf, + struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) { uint64_t ol_flags = mbuf->ol_flags; struct hinic3_offload_info *offload_info = &wqe_info->offload_info; @@ -442,6 +503,9 @@ hinic3_set_tx_offload(struct hinic3_nic_dev *nic_dev, offload_info->encapsulation = 1; wqe_info->queue_info.udp_dp_en = 1; break; + case HINIC3_PKT_TX_TUNNEL_IPIP: + offload_info->encapsulation = 1; + break; case 0: break; @@ -457,11 +521,113 @@ hinic3_set_tx_offload(struct hinic3_nic_dev *nic_dev, offload_info->out_l4_en = 1; set_tx_wqe_offload: - nic_dev->tx_ops->nic_tx_set_wqe_offload(wqe_info, wqe_combo); + hinic3_tx_set_compact_task_offload(wqe_info, wqe_combo); return 0; } +static int +hinic3_set_tx_offload(struct rte_mbuf *mbuf, + struct hinic3_sq_task *task, + struct hinic3_wqe_info *wqe_info, + enum nic_type nic_type) +{ + uint64_t ol_flags = mbuf->ol_flags; + uint16_t pld_offset = 0; + uint32_t queue_info = 0; + uint16_t vlan_tag; + + task->pkt_info0 = 0; + task->ip_identify = 0; + task->pkt_info2 = 0; + task->vlan_offload = 0; + + /* Vlan offload */ + if (unlikely(ol_flags & HINIC3_PKT_TX_VLAN_PKT)) { + vlan_tag = mbuf->vlan_tci; + hinic3_set_vlan_tx_offload(task, vlan_tag, HINIC3_TX_TPID0); + task->vlan_offload = hinic3_hw_be32(task->vlan_offload); + } + + if (!(ol_flags & HINIC3_TX_CKSUM_OFFLOAD_MASK)) + return 0; + + /* Tso offload */ + if (ol_flags & HINIC3_PKT_TX_TCP_SEG) { + if (((ol_flags & HINIC3_PKT_TX_TUNNEL_IPIP) == HINIC3_PKT_TX_TUNNEL_IPIP) && + nic_type == NIC_SP620) { + PMD_DRV_LOG(ERR, "IPinIP not support TSO"); + return -EINVAL; + } + if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) == HINIC3_PKT_TX_TUNNEL_VXLAN_GPE) { + PMD_DRV_LOG(ERR, "VXLAN_GPE not support TSO"); + return -EINVAL; + } + pld_offset = wqe_info->payload_offset; + if ((pld_offset >> 1) > MAX_PAYLOAD_OFFSET) + return -EINVAL; + + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN); + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN); + + queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, TSO); + queue_info |= SQ_CTRL_QUEUE_INFO_SET(pld_offset >> 1, PLDOFF); + + /* Set MSS value */ + queue_info = SQ_CTRL_QUEUE_INFO_CLEAR(queue_info, MSS); + queue_info |= SQ_CTRL_QUEUE_INFO_SET(mbuf->tso_segsz, MSS); + } else { + if (ol_flags & HINIC3_PKT_TX_IP_CKSUM) + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN); + + switch (ol_flags & HINIC3_PKT_TX_L4_MASK) { + case HINIC3_PKT_TX_TCP_CKSUM: + case HINIC3_PKT_TX_UDP_CKSUM: + case HINIC3_PKT_TX_SCTP_CKSUM: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN); + break; + case HINIC3_PKT_TX_L4_NO_CKSUM: + break; + default: + PMD_DRV_LOG(INFO, "not support pkt type"); + return -EINVAL; + } + } + + /* For vxlan, also can support PKT_TX_TUNNEL_GENEVE/GRE, etc */ + switch (ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) { + case HINIC3_PKT_TX_TUNNEL_VXLAN: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG); + break; + case HINIC3_PKT_TX_TUNNEL_VXLAN_GPE: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG); + break; + case HINIC3_PKT_TX_TUNNEL_GENEVE: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG); + break; + case HINIC3_PKT_TX_TUNNEL_IPIP: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG); + break; + case 0: + break; + default: + /* For non UDP/GRE tunneling, drop the tunnel packet */ + PMD_DRV_LOG(INFO, "not support tunnel pkt type"); + return -EINVAL; + } + + if (ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, OUT_L3_EN); + + if (ol_flags & HINIC3_PKT_TX_OUTER_UDP_CKSUM) + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, OUT_L4_EN); + + task->pkt_info0 = hinic3_hw_be32(task->pkt_info0); + task->pkt_info2 = hinic3_hw_be32(task->pkt_info2); + wqe_info->queue_info_sp600 = queue_info; + return 0; +} + /** * Check whether the number of segments in the mbuf is valid. * @@ -597,6 +763,8 @@ hinic3_non_tso_pkt_pre_process(struct rte_mbuf *mbuf, struct hinic3_wqe_info *wq * Point to the mbuf to send. * @param[in] wqe_info * Point to wqe_info of send queue(SQ). + * @param[in] non_tso_max_pkt_len + * Number of non_tso_max_pkt_len(SP230 is 9600, SP620/SP560 is 64K) * @return * 0 as success, -EINVAL as failure. */ @@ -802,6 +970,14 @@ hinic3_mbuf_dma_map_sge(struct hinic3_txq *txq, struct rte_mbuf *mbuf, hinic3_set_buf_desc(buf_desc, dma_addr, mbuf->data_len); buf_desc++; } + + /* + * SP620: For wqe compact type, no need to prepare + * sq ctrl info. Need set queue_info = 0. + */ + if (txq->nic_dev->nic_type == NIC_SP620) + wqe_desc->queue_info = 0; + mbuf = mbuf->next; } @@ -849,8 +1025,8 @@ hinic3_mbuf_dma_map_sge(struct hinic3_txq *txq, struct rte_mbuf *mbuf, * Point to wqe_info of send queue. */ static void -hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, - struct hinic3_wqe_info *wqe_info) +hinic3_prepare_sq_ctrl_compact_cqe(struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) { struct hinic3_queue_info *queue_info = &wqe_info->queue_info; struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr; @@ -863,7 +1039,7 @@ hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, if (wqe_combo->wqe_type == SQ_WQE_EXTENDED_TYPE) { wqe_desc->ctrl_len |= SQ_CTRL_SET(wqe_info->sge_cnt, BUFDESC_NUM) | SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) | - SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT); + SQ_CTRL_SET(SQ_WQE_SGL, DATA_FORMAT); *qsf = SQ_CTRL_QUEUE_INFO_SET(1, UC) | SQ_CTRL_QUEUE_INFO_SET(queue_info->sctp, SCTP) | @@ -884,14 +1060,56 @@ hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, *qsf = hinic3_hw_be32(*qsf); } else { wqe_desc->ctrl_len |= SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->sctp, SCTP) | - SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->udp_dp_en, UDP_DP_EN) | - SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->ufo, UFO) | - SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->pkt_type, PKT_TYPE); + SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->udp_dp_en, + UDP_DP_EN) | + SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->ufo, UFO) | + SQ_CTRL_COMPACT_QUEUE_INFO_SET(queue_info->pkt_type, + PKT_TYPE); } wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len); } +/** + * Sets and configures fields in the transmit queue control descriptor based on + * the WQE type. + * + * @param[out] wqe_combo + * Point to wqe_combo of send queue. + * @param[in] wqe_info + * Point to wqe_info of send queue. + */ +static void +hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) +{ + struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr; + + wqe_desc->ctrl_len |= SQ_CTRL_SET(wqe_info->sge_cnt, BUFDESC_NUM) | + SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) | + SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | + SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) | + SQ_CTRL_SET(wqe_info->owner, OWNER); + + wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len); + + wqe_desc->queue_info = wqe_info->queue_info_sp600; + wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, UC); + + if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) { + wqe_desc->queue_info |= + SQ_CTRL_QUEUE_INFO_SET(TX_MSS_DEFAULT, MSS); + } else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) < + TX_MSS_MIN) { + /* Mss should not less than 80 */ + wqe_desc->queue_info = + SQ_CTRL_QUEUE_INFO_CLEAR(wqe_desc->queue_info, MSS); + wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(TX_MSS_MIN, MSS); + } + + wqe_desc->queue_info = hinic3_hw_be32(wqe_desc->queue_info); +} + /** * It is responsible for sending data packets. * @@ -911,10 +1129,11 @@ hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) struct hinic3_tx_info *tx_info = NULL; struct rte_mbuf *mbuf_pkt = NULL; struct hinic3_sq_wqe_combo wqe_combo = {0}; + struct hinic3_sq_wqe *sq_wqe = NULL; struct hinic3_wqe_info wqe_info = {0}; uint32_t offload_err, free_cnt; uint64_t tx_bytes = 0; - uint16_t free_wqebb_cnt, nb_tx; + uint16_t free_wqebb_cnt, nb_tx, nb_pkt = 0; int err; #ifdef HINIC3_XSTAT_PROF_TX @@ -925,6 +1144,7 @@ hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) if (unlikely(!HINIC3_TXQ_IS_STARTED(txq))) return 0; + rte_prefetch0(tx_pkts[0]); free_cnt = txq->tx_free_thresh; /* Reclaim tx mbuf before xmit new packets. */ if (hinic3_get_sq_free_wqebbs(txq) < txq->tx_free_thresh) @@ -933,59 +1153,73 @@ hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Tx loop routine. */ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { mbuf_pkt = *tx_pkts++; + rte_prefetch0(*tx_pkts); if (unlikely(hinic3_get_tx_offload(txq->nic_dev, mbuf_pkt, &wqe_info, txq->non_tso_max_pkt_len))) { + rte_pktmbuf_free(mbuf_pkt); txq->txq_stats.offload_errors++; - break; + continue; } - wqe_info.wqebb_cnt = wqe_info.sge_cnt; - if (likely(wqe_info.offload || wqe_info.wqebb_cnt > 1)) { - if (txq->tx_wqe_compact_task) { - /** - * One more wqebb is needed for compact task under two situations: - * 1. TSO: MSS field is needed, no available space for - * compact task in compact wqe. - * 2. SGE number > 1: wqe is handlerd as extended wqe by nic. - */ - if (mbuf_pkt->ol_flags & HINIC3_PKT_TX_TCP_SEG || - wqe_info.wqebb_cnt > 1) - wqe_info.wqebb_cnt++; - } else { - /* Use extended sq wqe with normal TS */ - wqe_info.wqebb_cnt++; - } - } + if (!wqe_info.offload) + /* + * Use extended sq wqe with small TS, which can include + * multi sges, or compact sq normal wqe, which just + * supports one sge + */ + wqe_info.wqebb_cnt = wqe_info.sge_cnt; + else + /* Use extended sq wqe with normal TS */ + wqe_info.wqebb_cnt = wqe_info.sge_cnt + 1; free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); - if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) { - /* Reclaim again. */ + if (wqe_info.wqebb_cnt > free_wqebb_cnt) { + /* + * Try to reclaim completed Tx WQEs when SQ space is + * insufficient. This gives hardware a chance to free + * descriptors and helps prevent packet drops caused + * by transient SQ congestion. + */ hinic3_xmit_mbuf_cleanup(txq, free_cnt); free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); - if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) { + if (wqe_info.wqebb_cnt > free_wqebb_cnt) { txq->txq_stats.tx_busy += (nb_pkts - nb_tx); - break; + goto end; } } - /* Task or bd section maybe wrapped for one wqe. */ - hinic3_set_wqe_combo(txq, &wqe_combo, &wqe_info); + /* Get sq wqe address from wqe_page */ + sq_wqe = hinic3_get_sq_wqe(txq, &wqe_info); + if (unlikely(!sq_wqe)) { + txq->txq_stats.tx_busy++; + break; + } - /* Fill tx packet offload into qsf and task field. */ - offload_err = hinic3_set_tx_offload(txq->nic_dev, mbuf_pkt, &wqe_combo, &wqe_info); + /* Task or bd section maybe wrapped for one wqe */ + hinic3_set_wqe_combo(txq, &wqe_combo, sq_wqe, &wqe_info); + + wqe_info.queue_info_sp600 = 0; + /* Fill tx packet offload into qsf and task field */ + if (wqe_info.offload) { + offload_err = hinic3_set_tx_offload(mbuf_pkt, + wqe_combo.task, + &wqe_info, + txq->nic_dev->nic_type); if (unlikely(offload_err)) { hinic3_put_sq_wqe(txq, &wqe_info); + rte_pktmbuf_free(mbuf_pkt); txq->txq_stats.offload_errors++; - break; + continue; } - + } /* Fill sq_wqe buf_desc and bd_desc. */ err = hinic3_mbuf_dma_map_sge(txq, mbuf_pkt, &wqe_combo, &wqe_info); if (err) { hinic3_put_sq_wqe(txq, &wqe_info); + rte_pktmbuf_free(mbuf_pkt); txq->txq_stats.offload_errors++; - break; + continue; } /* Record tx info. */ @@ -1001,27 +1235,28 @@ hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) hinic3_prepare_sq_ctrl(&wqe_combo, &wqe_info); tx_bytes += mbuf_pkt->pkt_len; + nb_pkt++; } - +end: /* Update txq stats. */ - if (nb_tx) { + if (nb_pkt) { hinic3_write_db(txq->db_addr, txq->q_id, (int)(txq->cos), SQ_CFLAG_DP, MASKED_QUEUE_IDX(txq, txq->prod_idx)); - txq->txq_stats.packets += nb_tx; + txq->txq_stats.packets += nb_pkt; txq->txq_stats.bytes += tx_bytes; } - txq->txq_stats.burst_pkts = nb_tx; + txq->txq_stats.burst_pkts = nb_pkt; #ifdef HINIC3_XSTAT_PROF_TX t2 = rte_get_tsc_cycles(); txq->txq_stats.app_tsc = t1 - txq->prof_tx_end_tsc; txq->prof_tx_end_tsc = t2; txq->txq_stats.pmd_tsc = t2 - t1; - txq->txq_stats.burst_pkts = nb_tx; + txq->txq_stats.burst_pkts = nb_pkt; #endif - return nb_tx; + return nb_pkt; } int @@ -1077,3 +1312,139 @@ hinic3_flush_txqs(struct hinic3_nic_dev *nic_dev) PMD_DRV_LOG(ERR, "Stop sq%d failed", qid); } } + +/** + * It is responsible for sending data packets. + * + * @param[in] tx_queue + * Point to send queue. + * @param[in] tx_pkts + * Pointer to the array of data packets to be sent. + * @param[in] nb_pkts + * Number of sent packets. + * @return + * Number of actually sent packets. + */ +uint16_t +hinic3_xmit_pkts_compact_cqe(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct hinic3_txq *txq = tx_queue; + struct hinic3_nic_dev *nic_dev = txq->nic_dev; + struct hinic3_tx_info *tx_info = NULL; + struct rte_mbuf *mbuf_pkt = NULL; + struct hinic3_sq_wqe_combo wqe_combo = {0}; + struct hinic3_wqe_info wqe_info = {0}; + uint32_t offload_err, free_cnt; + uint64_t tx_bytes = 0; + uint16_t free_wqebb_cnt, nb_tx, nb_pkt = 0; + int err; + +#ifdef HINIC3_XSTAT_PROF_TX + uint64_t t1, t2; + t1 = rte_get_tsc_cycles(); +#endif + + if (unlikely(!HINIC3_TXQ_IS_STARTED(txq))) + return 0; + + free_cnt = txq->tx_free_thresh; + /* Reclaim tx mbuf before xmit new packets. */ + if (hinic3_get_sq_free_wqebbs(txq) < txq->tx_free_thresh) + hinic3_xmit_mbuf_cleanup(txq, free_cnt); + + /* Tx loop routine. */ + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + mbuf_pkt = *tx_pkts++; + if (unlikely(hinic3_get_tx_offload(nic_dev, mbuf_pkt, &wqe_info, + txq->non_tso_max_pkt_len))) { + rte_pktmbuf_free(mbuf_pkt); + txq->txq_stats.offload_errors++; + continue; + } + + wqe_info.wqebb_cnt = wqe_info.sge_cnt; + if (likely(wqe_info.offload || wqe_info.wqebb_cnt > 1)) { + if (txq->tx_wqe_compact_task) { + /** + * One more wqebb is needed for compact task under two situations: + * 1. TSO: MSS field is needed, no available space for + * compact task in compact wqe. + * 2. SGE number > 1: wqe is handlerd as extended wqe by nic. + */ + if (mbuf_pkt->ol_flags & HINIC3_PKT_TX_TCP_SEG || + wqe_info.wqebb_cnt > 1) + wqe_info.wqebb_cnt++; + } else { + /* Use extended sq wqe with normal TS */ + wqe_info.wqebb_cnt++; + } + } + + free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); + if (wqe_info.wqebb_cnt > free_wqebb_cnt) { + /* Reclaim again */ + hinic3_xmit_mbuf_cleanup(txq, free_cnt); + free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); + if (wqe_info.wqebb_cnt > free_wqebb_cnt) { + txq->txq_stats.tx_busy += (nb_pkts - nb_tx); + goto end; + } + } + + /* Task or bd section maybe wrapped for one wqe. */ + hinic3_set_wqe_combo_compact_cqe(txq, &wqe_combo, &wqe_info); + + /* Fill tx packet offload into qsf and task field. */ + offload_err = hinic3_set_tx_offload_compact_cqe(mbuf_pkt, &wqe_combo, &wqe_info); + if (unlikely(offload_err)) { + hinic3_put_sq_wqe(txq, &wqe_info); + rte_pktmbuf_free(mbuf_pkt); + txq->txq_stats.offload_errors++; + continue; + } + + /* Fill sq_wqe buf_desc and bd_desc. */ + err = hinic3_mbuf_dma_map_sge(txq, mbuf_pkt, &wqe_combo, &wqe_info); + + if (err) { + hinic3_put_sq_wqe(txq, &wqe_info); + rte_pktmbuf_free(mbuf_pkt); + txq->txq_stats.offload_errors++; + continue; + } + + /* Record tx info. */ + tx_info = &txq->tx_info[wqe_info.pi]; + tx_info->mbuf = mbuf_pkt; + tx_info->wqebb_cnt = wqe_info.wqebb_cnt; + + /* + * For wqe compact type, no need to prepare + * sq ctrl info. + */ + hinic3_prepare_sq_ctrl_compact_cqe(&wqe_combo, &wqe_info); + + tx_bytes += mbuf_pkt->pkt_len; + nb_pkt++; + } +end: + /* Update txq stats. */ + if (nb_pkt) { + hinic3_write_db(txq->db_addr, txq->q_id, (int)(txq->cos), + SQ_CFLAG_DP, + MASKED_QUEUE_IDX(txq, txq->prod_idx)); + txq->txq_stats.packets += nb_pkt; + txq->txq_stats.bytes += tx_bytes; + } + txq->txq_stats.burst_pkts = nb_pkt; + +#ifdef HINIC3_XSTAT_PROF_TX + t2 = rte_get_tsc_cycles(); + txq->txq_stats.app_tsc = t1 - txq->prof_tx_end_tsc; + txq->prof_tx_end_tsc = t2; + txq->txq_stats.pmd_tsc = t2 - t1; + txq->txq_stats.burst_pkts = nb_pkt; +#endif + + return nb_pkt; +} diff --git a/drivers/net/hinic3/hinic3_tx.h b/drivers/net/hinic3/hinic3_tx.h index 5d4101630e..b216a39ecd 100644 --- a/drivers/net/hinic3/hinic3_tx.h +++ b/drivers/net/hinic3/hinic3_tx.h @@ -55,10 +55,10 @@ struct hinic3_wqe_info { uint16_t sge_cnt; uint8_t offload; - uint8_t rsvd0; /**< Reserved field 0. */ + uint8_t wrapped; uint16_t payload_offset; - uint8_t rsvd1; /**< Reserved field 1. */ + uint32_t queue_info_sp600; uint8_t owner; uint16_t pi; @@ -388,41 +388,12 @@ struct __rte_cache_aligned hinic3_txq { #endif }; -/* Tx WQE offload set callback function */ -typedef void (*nic_tx_set_wqe_offload_t)(struct hinic3_wqe_info *wqe_info, - struct hinic3_sq_wqe_combo *wqe_combo); - -struct hinic3_nic_tx_ops { - nic_tx_set_wqe_offload_t nic_tx_set_wqe_offload; -}; - void hinic3_flush_txqs(struct hinic3_nic_dev *nic_dev); void hinic3_free_txq_mbufs(struct hinic3_txq *txq); void hinic3_free_all_txq_mbufs(struct hinic3_nic_dev *nic_dev); +uint16_t hinic3_xmit_pkts_compact_cqe(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int hinic3_stop_sq(struct hinic3_txq *txq); int hinic3_start_all_sqs(struct rte_eth_dev *eth_dev); int hinic3_tx_done_cleanup(void *txq, uint32_t free_cnt); - -/** - * Set wqe task section - * - * @param[in] wqe_info - * Packet info parsed according to mbuf - * @param[in] wqe_combo - * Wqe need to format - */ -void hinic3_tx_set_normal_task_offload(struct hinic3_wqe_info *wqe_info, - struct hinic3_sq_wqe_combo *wqe_combo); - -/** - * Set compact wqe task section - * - * @param[in] wqe_info - * Packet info parsed according to mbuf - * @param[in] wqe_combo - * Wqe need to format - */ -void hinic3_tx_set_compact_task_offload(struct hinic3_wqe_info *wqe_info, - struct hinic3_sq_wqe_combo *wqe_combo); #endif /**< _HINIC3_TX_H_ */ -- 2.33.0
