This commit introduces new functions to support direct data placement
(DDP) NIC offloads that avoid copying data from SKBs.
Later patches will use this for nvme-tcp DDP offload.
Signed-off-by: Boris Pismenny <bor...@mellanox.com>
Signed-off-by: Ben Ben-Ishay <benis...@nvidia.com>
Signed-off-by: Or Gerlitz <ogerl...@mellanox.com>
Signed-off-by: Yoray Zack <yor...@mellanox.com>
---
include/linux/skbuff.h | 9 ++++++++
net/core/datagram.c | 48 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 57 insertions(+)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 0d1be25574cc..0470eb3c9976 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3594,6 +3594,10 @@ __poll_t datagram_poll(struct file *file, struct socket
*sock,
struct poll_table_struct *wait);
int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
struct iov_iter *to, int size);
+#ifdef CONFIG_TCP_DDP
+int skb_ddp_copy_datagram_iter(const struct sk_buff *from, int offset,
+ struct iov_iter *to, int size);
+#endif
static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
struct msghdr *msg, int size)
{
@@ -3604,6 +3608,11 @@ int skb_copy_and_csum_datagram_msg(struct sk_buff *skb,
int hlen,
int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len,
struct ahash_request *hash);
+#ifdef CONFIG_TCP_DDP
+int skb_ddp_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len,
+ struct ahash_request *hash);
+#endif
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
struct iov_iter *from, int len);
int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 81809fa735a7..f386d4d82b1b 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -495,6 +495,27 @@ static int __skb_datagram_iter(const struct sk_buff *skb,
int offset,
return 0;
}
+#ifdef CONFIG_TCP_DDP
+/**
+ * skb_ddp_copy_and_hash_datagram_iter - Copies datagrams from skb frags to
+ * an iterator and update a hash. If the iterator and skb frag point to the
+ * same page and offset, then the copy is skipped.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ * @hash: hash request to update
+ */
+int skb_ddp_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len,
+ struct ahash_request *hash)
+{
+ return __skb_datagram_iter(skb, offset, to, len, true,
+ ddp_hash_and_copy_to_iter, hash);
+}
+EXPORT_SYMBOL(skb_ddp_copy_and_hash_datagram_iter);
+#endif
+
/**
* skb_copy_and_hash_datagram_iter - Copy datagram to an iovec iterator
* and update a hash.
@@ -513,6 +534,33 @@ int skb_copy_and_hash_datagram_iter(const struct sk_buff
*skb, int offset,
}
EXPORT_SYMBOL(skb_copy_and_hash_datagram_iter);
+#ifdef CONFIG_TCP_DDP
+static size_t simple_ddp_copy_to_iter(const void *addr, size_t bytes,
+ void *data __always_unused,
+ struct iov_iter *i)
+{
+ return ddp_copy_to_iter(addr, bytes, i);
+}
+
+/**
+ * skb_ddp_copy_datagram_iter - Copies datagrams from skb frags to an
+ * iterator. If the iterator and skb frag point to the same page and
+ * offset, then the copy is skipped.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ */
+int skb_ddp_copy_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len)
+{
+ trace_skb_copy_datagram_iovec(skb, len);
+ return __skb_datagram_iter(skb, offset, to, len, false,
+ simple_ddp_copy_to_iter, NULL);
+}
+EXPORT_SYMBOL(skb_ddp_copy_datagram_iter);
+#endif
+
static size_t simple_copy_to_iter(const void *addr, size_t bytes,
void *data __always_unused, struct iov_iter *i)
{
--
2.24.1
