When using direct data placement the NIC writes some of the payload directly to the destination buffer, and constructs the SKB such that it points to this data. As a result, the skb_copy datagram_iter call will attempt to copy data when it is not necessary.
This patch adds a check to avoid this copy, and a static_key to enabled it when TCP direct data placement is possible. Signed-off-by: Boris Pismenny <bor...@mellanox.com> Signed-off-by: Ben Ben-Ishay <benis...@mellanox.com> Signed-off-by: Or Gerlitz <ogerl...@mellanox.com> Signed-off-by: Yoray Zack <yor...@mellanox.com> --- include/linux/uio.h | 2 ++ lib/iov_iter.c | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/include/linux/uio.h b/include/linux/uio.h index 72d88566694e..05573d848ff5 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -282,4 +282,6 @@ int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, int (*f)(struct kvec *vec, void *context), void *context); +extern struct static_key_false skip_copy_enabled; + #endif diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 1635111c5bd2..206edb051135 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -15,6 +15,9 @@ #define PIPE_PARANOIA /* for now */ +DEFINE_STATIC_KEY_FALSE(skip_copy_enabled); +EXPORT_SYMBOL_GPL(skip_copy_enabled); + #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ size_t left; \ size_t wanted = n; \ @@ -476,7 +479,13 @@ static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) { char *to = kmap_atomic(page); - memcpy(to + offset, from, len); + + if (static_branch_unlikely(&skip_copy_enabled)) { + if (to + offset != from) + memcpy(to + offset, from, len); + } else { + memcpy(to + offset, from, len); + } kunmap_atomic(to); } -- 2.24.1