When using direct data placement the NIC writes some of the payload
directly to the destination buffer, and constructs the SKB such that it
points to this data. As a result, the skb_copy datagram_iter call will
attempt to copy data when it is not necessary.
This patch adds a check to avoid this copy, and a static_key to enabled
it when TCP direct data placement is possible.
Signed-off-by: Boris Pismenny <bor...@mellanox.com>
Signed-off-by: Ben Ben-Ishay <benis...@mellanox.com>
Signed-off-by: Or Gerlitz <ogerl...@mellanox.com>
Signed-off-by: Yoray Zack <yor...@mellanox.com>
---
include/linux/uio.h | 2 ++
lib/iov_iter.c | 11 ++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 3835a8a8e9ea..036df4073737 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -284,4 +284,6 @@ int iov_iter_for_each_range(struct iov_iter *i, size_t
bytes,
int (*f)(struct kvec *vec, void *context),
void *context);
+extern struct static_key_false skip_copy_enabled;
+
#endif
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 5e40786c8f12..27856c3db28b 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -13,6 +13,9 @@
#define PIPE_PARANOIA /* for now */
+DEFINE_STATIC_KEY_FALSE(skip_copy_enabled);
+EXPORT_SYMBOL_GPL(skip_copy_enabled);
+
#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
size_t left; \
size_t wanted = n; \
@@ -470,7 +473,13 @@ static void memcpy_from_page(char *to, struct page *page,
size_t offset, size_t
static void memcpy_to_page(struct page *page, size_t offset, const char *from,
size_t len)
{
char *to = kmap_atomic(page);
- memcpy(to + offset, from, len);
+
+ if (static_branch_unlikely(&skip_copy_enabled)) {
+ if (to + offset != from)
+ memcpy(to + offset, from, len);
+ } else {
+ memcpy(to + offset, from, len);
+ }
kunmap_atomic(to);
}
--
2.24.1
