On 2026/7/9 19:25, Li Zhe wrote:
Introduce memcpy_nt() and memcpy_nt_drain() for write-once copy sites
that want a named non-temporal copy primitive plus an explicit drain
step.
On x86_64, override both helpers in arch/x86/include/asm/string_64.h
using the usual self-macro pattern, next to the existing
memcpy_flushcache() backend that memcpy_nt() wraps. The x86_64
implementation maps memcpy_nt() to memcpy_flushcache() and uses wmb()
for memcpy_nt_drain(), because that backend issues MOVNTI stores and
callers need an ordering point before later normal stores that depend
on those writes becoming visible.
include/linux/string.h provides the generic fallback under
memcpy_nt() as plain memcpy() and leaves memcpy_nt_drain() empty, so
architectures that do not override memcpy_nt() do not pay an
unconditional barrier. Architectures that later grow a specialized
memcpy_nt() backend can override memcpy_nt_drain() with whatever
drain primitive their memory-ordering rules require.
The immediate user is the ZONE_DEVICE template-copy path. It populates
struct page descriptors in a write-once pattern, so a regular cached
memcpy() can incur avoidable write-allocate traffic and cache
pollution for data with little near-term reuse.
Signed-off-by: Li Zhe <[email protected]>
---
arch/x86/include/asm/string_64.h | 22 ++++++++++++++++++++++
include/linux/string.h | 23 +++++++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 4635616863f5..6cb9e0ac7fa0 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -4,6 +4,7 @@
#ifdef __KERNEL__
#include <linux/jump_label.h>
+#include <asm/barrier.h>
/* Written 2002 by Andi Kleen */
@@ -100,6 +101,27 @@ static __always_inline void memcpy_flushcache(void *dst,
const void *src, size_t
}
__memcpy_flushcache(dst, src, cnt);
}
+
+#define memcpy_nt memcpy_nt
+/*
+ * Reuse the existing x86 flushcache backend as the nt copy primitive.
+ * Callers pair it with memcpy_nt_drain() when later stores must be
+ * ordered after the copy.
+ */
+static __always_inline void memcpy_nt(void *dst, const void *src, size_t cnt)
+{
+ memcpy_flushcache(dst, src, cnt);
Why not use memcpy_flushcache() directly in device dax path? I don't
understand the necessity of introducing memcpy_nt here.
+}
+
+#define memcpy_nt_drain memcpy_nt_drain
+static __always_inline void memcpy_nt_drain(void)
+{
+ /*
+ * Order the prior MOVNTI stores issued by memcpy_flushcache()
+ * before later normal stores.
+ */
I also have a question here: why are we using wmb to guarantee visibility
at this stage?
Since we are still in the very early phases of memory initialization
(specifically,
struct page initialization), since we are still in an intermediate
initialization
state, this shouldn't be visible to other CPUs anyway.
Thanks.
+ wmb();
+}
#endif
#endif /* __KERNEL__ */
diff --git a/include/linux/string.h b/include/linux/string.h
index 5702daca4326..a109b2f86ca6 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -278,6 +278,29 @@ static inline void memcpy_flushcache(void *dst, const void
*src, size_t cnt)
}
#endif
+#ifndef memcpy_nt
+/*
+ * memcpy_nt() requests a non-temporal copy when the architecture has a
+ * suitable backend. Architectures that do not override it fall back to
+ * memcpy().
+ */
+static inline void memcpy_nt(void *dst, const void *src, size_t cnt)
+{
+ memcpy(dst, src, cnt);
+}
+#endif
+
+#ifndef memcpy_nt_drain
+/*
+ * Callers use memcpy_nt_drain() before later normal stores that need to
+ * be ordered after memcpy_nt(). Architectures without a specialized
+ * backend can leave it empty.
+ */
+static inline void memcpy_nt_drain(void)
+{
+}
+#endif
+
void *memchr_inv(const void *s, int c, size_t n);
char *strreplace(char *str, char old, char new);
--
2.20.1
