From: Yoray Zack <yor...@mellanox.com>
crc offload of the nvme capsule. Check if all the skb bits
are on, and if not recalculate the crc in SW and check it.
This patch reworks the receive-side crc calculation to always
run at the end, so as to keep a single flow for both offload
and non-offload. This change simplifies the code, but it may degrade
performance for non-offload crc calculation.
Signed-off-by: Boris Pismenny <bor...@mellanox.com>
Signed-off-by: Ben Ben-Ishay <benis...@mellanox.com>
Signed-off-by: Or Gerlitz <ogerl...@mellanox.com>
Signed-off-by: Yoray Zack <yor...@mellanox.com>
---
drivers/nvme/host/tcp.c | 66 ++++++++++++++++++++++++++++++++++++-----
1 file changed, 58 insertions(+), 8 deletions(-)
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 7bd97f856677..9a620d1dacb4 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -94,6 +94,7 @@ struct nvme_tcp_queue {
size_t data_remaining;
size_t ddgst_remaining;
unsigned int nr_cqe;
+ bool crc_valid;
/* send state */
struct nvme_tcp_request *request;
@@ -233,6 +234,41 @@ static inline size_t nvme_tcp_pdu_last_send(struct
nvme_tcp_request *req,
return nvme_tcp_pdu_data_left(req) <= len;
}
+static inline bool nvme_tcp_device_ddgst_ok(struct nvme_tcp_queue *queue)
+{
+ return queue->crc_valid;
+}
+
+static inline void nvme_tcp_device_ddgst_update(struct nvme_tcp_queue *queue,
+ struct sk_buff *skb)
+{
+ if (queue->crc_valid)
+#ifdef CONFIG_TCP_DDP_CRC
+ queue->crc_valid = skb->ddp_crc;
+#else
+ queue->crc_valid = false;
+#endif
+}
+
+static void nvme_tcp_crc_recalculate(struct nvme_tcp_queue *queue,
+ struct nvme_tcp_data_pdu *pdu)
+{
+ struct nvme_tcp_request *req;
+ struct request *rq;
+
+ rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
+ if (!rq)
+ return;
+ req = blk_mq_rq_to_pdu(rq);
+ crypto_ahash_init(queue->rcv_hash);
+ req->ddp.sg_table.sgl = req->ddp.first_sgl;
+ /* req->ddp.sg_table is allocated and filled in nvme_tcp_setup_ddp */
+ ahash_request_set_crypt(queue->rcv_hash, req->ddp.sg_table.sgl, NULL,
+ le32_to_cpu(pdu->data_length));
+ crypto_ahash_update(queue->rcv_hash);
+}
+
+
#ifdef CONFIG_TCP_DDP
bool nvme_tcp_resync_request(struct sock *sk, u32 seq, u32 flags);
@@ -706,6 +742,7 @@ static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue
*queue)
queue->pdu_offset = 0;
queue->data_remaining = -1;
queue->ddgst_remaining = 0;
+ queue->crc_valid = true;
}
static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
@@ -955,6 +992,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue,
struct sk_buff *skb,
struct nvme_tcp_request *req;
struct request *rq;
+ if (test_bit(NVME_TCP_Q_OFFLOADS, &queue->flags))
+ nvme_tcp_device_ddgst_update(queue, skb);
rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
if (!rq) {
dev_err(queue->ctrl->ctrl.device,
@@ -992,7 +1031,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue
*queue, struct sk_buff *skb,
recv_len = min_t(size_t, recv_len,
iov_iter_count(&req->iter));
- if (queue->data_digest)
+ if (queue->data_digest && !test_bit(NVME_TCP_Q_OFFLOADS,
&queue->flags))
ret = skb_copy_and_hash_datagram_iter(skb, *offset,
&req->iter, recv_len, queue->rcv_hash);
else
@@ -1012,7 +1051,6 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue
*queue, struct sk_buff *skb,
if (!queue->data_remaining) {
if (queue->data_digest) {
- nvme_tcp_ddgst_final(queue->rcv_hash,
&queue->exp_ddgst);
queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
} else {
if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
@@ -1033,8 +1071,11 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue
*queue,
char *ddgst = (char *)&queue->recv_ddgst;
size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
+ bool ddgst_offload_fail;
int ret;
+ if (test_bit(NVME_TCP_Q_OFFLOADS, &queue->flags))
+ nvme_tcp_device_ddgst_update(queue, skb);
ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
if (unlikely(ret))
return ret;
@@ -1045,12 +1086,21 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue
*queue,
if (queue->ddgst_remaining)
return 0;
- if (queue->recv_ddgst != queue->exp_ddgst) {
- dev_err(queue->ctrl->ctrl.device,
- "data digest error: recv %#x expected %#x\n",
- le32_to_cpu(queue->recv_ddgst),
- le32_to_cpu(queue->exp_ddgst));
- return -EIO;
+ ddgst_offload_fail = !nvme_tcp_device_ddgst_ok(queue);
+ if (!test_bit(NVME_TCP_Q_OFFLOADS, &queue->flags) ||
+ ddgst_offload_fail) {
+ if (test_bit(NVME_TCP_Q_OFFLOADS, &queue->flags) &&
+ ddgst_offload_fail)
+ nvme_tcp_crc_recalculate(queue, pdu);
+
+ nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
+ if (queue->recv_ddgst != queue->exp_ddgst) {
+ dev_err(queue->ctrl->ctrl.device,
+ "data digest error: recv %#x expected %#x\n",
+ le32_to_cpu(queue->recv_ddgst),
+ le32_to_cpu(queue->exp_ddgst));
+ return -EIO;
+ }
}
if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
--
2.24.1
