Some devices want to hotplug their memory onto a node, but not have it
exposed as "general purpose". The intent is to re-use portions of mm/
services for management without exposing it for general consumption.
N_MEMORY nodes are intended to contain general System RAM, so they
are unsuited for this purpose.
Create N_MEMORY_PRIVATE for memory nodes whose memory is not intended
for general consumption.
N_MEMORY and N_MEMORY_PRIVATE are mutually exclusive states.
This commit adds basic infrastructure for N_MEMORY_PRIVATE nodes:
- struct node_private:
Per-node container stored in NODE_DATA(nid) storing the owner.
- folio/page_is_private_node():
true if it resides on a private node
- folio/page_is_private_managed():
common predicate for checking private node or zone device
Used to combine common filtering locations.
- Registration API: node_private_register()/unregister()
- sysfs attribute exposing N_MEMORY_PRIVATE node state.
- fix node_state() and helpers to return false for N_MEMORY_PRIVATE
when !CONFIG_NUMA (mutually exclusive w/ N_MEMORY)
Signed-off-by: Gregory Price <[email protected]>
---
Documentation/ABI/stable/sysfs-devices-node | 10 ++
drivers/base/node.c | 113 ++++++++++++++++++++
include/linux/mmzone.h | 15 +++
include/linux/node_private.h | 82 ++++++++++++++
include/linux/nodemask.h | 7 +-
mm/internal.h | 12 +++
6 files changed, 236 insertions(+), 3 deletions(-)
create mode 100644 include/linux/node_private.h
diff --git a/Documentation/ABI/stable/sysfs-devices-node
b/Documentation/ABI/stable/sysfs-devices-node
index 2d0e023f22a71..8a36fbc3d4028 100644
--- a/Documentation/ABI/stable/sysfs-devices-node
+++ b/Documentation/ABI/stable/sysfs-devices-node
@@ -29,6 +29,16 @@ Description:
Nodes that have regular or high memory.
Depends on CONFIG_HIGHMEM.
+What: /sys/devices/system/node/has_private_memory
+Contact: Linux Memory Management list <[email protected]>
+Description:
+ Nodes that have private (N_MEMORY_PRIVATE) memory: memory
+ hotplugged by a driver onto a CPU-less node and isolated from
+ the page allocator's normal and fallback zonelists. A node is
+ listed here for as long as it holds private memory; it is never
+ listed in has_memory at the same time (the two states are
+ mutually exclusive). See
Documentation/mm/numa_private_nodes.rst.
+
What: /sys/devices/system/node/nodeX
Date: October 2002
Contact: Linux Memory Management list <[email protected]>
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 3da91929ad4e3..94cd51f51b7e8 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -22,6 +22,7 @@
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/memblock.h>
+#include <linux/node_private.h>
static const struct bus_type node_subsys = {
.name = "node",
@@ -868,6 +869,116 @@ void register_memory_blocks_under_node_hotplug(int nid,
unsigned long start_pfn,
(void *)&nid, register_mem_block_under_node_hotplug);
return;
}
+
+static DEFINE_MUTEX(node_private_lock);
+
+/**
+ * node_private_register - Register a private node
+ * @nid: Node identifier
+ * @np: The node_private structure (driver-allocated, driver-owned)
+ *
+ * Register an owner for a private node. If an owner is already registered
+ * (different np), return -EBUSY.
+ *
+ * Re-registration with the same np is allowed.
+ *
+ * The caller owns the node_private memory and must ensure it remains valid
+ * until after node_private_unregister() returns.
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+int node_private_register(int nid, struct node_private *np)
+{
+ struct node_private *existing;
+ pg_data_t *pgdat;
+ int ret = 0;
+
+ if (!np || !node_possible(nid))
+ return -EINVAL;
+
+ mutex_lock(&node_private_lock);
+ mem_hotplug_begin();
+
+ /* N_MEMORY_PRIVATE and N_MEMORY are mutually exclusive */
+ if (node_state(nid, N_MEMORY)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ pgdat = NODE_DATA(nid);
+ existing = rcu_dereference_protected(pgdat->node_private,
+
lockdep_is_held(&node_private_lock));
+
+ /* If it exists, restrict to single owner */
+ if (existing) {
+ if (existing != np)
+ ret = -EBUSY;
+ goto out;
+ }
+
+ rcu_assign_pointer(pgdat->node_private, np);
+out:
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(node_private_register);
+
+/**
+ * node_private_unregister - Unregister a private node
+ * @nid: Node identifier
+ *
+ * Unregister the driver from a private node.
+ *
+ * Only succeeds if all memory has been offlined (N_MEMORY_PRIVATE cleared).
+ *
+ * N_MEMORY_PRIVATE state is cleared by offline_pages() when the last
+ * memory is offlined, not by this function.
+ *
+ * After this returns, the node_private pointer is no longer visible and
+ * an RCU grace period has elapsed, so the driver may free its context.
+ *
+ * Return: 0 if unregistered, -EBUSY if N_MEMORY_PRIVATE is still set.
+ */
+int node_private_unregister(int nid)
+{
+ struct node_private *np;
+ pg_data_t *pgdat;
+
+ if (!node_possible(nid))
+ return 0;
+
+ mutex_lock(&node_private_lock);
+ mem_hotplug_begin();
+
+ pgdat = NODE_DATA(nid);
+ np = rcu_dereference_protected(pgdat->node_private,
+ lockdep_is_held(&node_private_lock));
+ if (!np) {
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return 0;
+ }
+
+ /*
+ * Only unregister if N_MEMORY_PRIVATE is cleared (only occurs when
+ * the last memory block is offlined in offline_pages())
+ */
+ if (node_is_private(nid)) {
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+ return -EBUSY;
+ }
+
+ rcu_assign_pointer(pgdat->node_private, NULL);
+
+ mem_hotplug_done();
+ mutex_unlock(&node_private_lock);
+
+ synchronize_rcu();
+ return 0;
+}
+EXPORT_SYMBOL_GPL(node_private_unregister);
#endif /* CONFIG_MEMORY_HOTPLUG */
/**
@@ -966,6 +1077,7 @@ static struct node_attr node_state_attr[] = {
[N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
#endif
[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
+ [N_MEMORY_PRIVATE] = _NODE_ATTR(has_private_memory, N_MEMORY_PRIVATE),
[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
N_GENERIC_INITIATOR),
@@ -979,6 +1091,7 @@ static struct attribute *node_state_attrs[] = {
&node_state_attr[N_HIGH_MEMORY].attr.attr,
#endif
&node_state_attr[N_MEMORY].attr.attr,
+ &node_state_attr[N_MEMORY_PRIVATE].attr.attr,
&node_state_attr[N_CPU].attr.attr,
&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
NULL
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0507193b3ae34..9815e48c03b97 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -26,6 +26,8 @@
#include <linux/sizes.h>
#include <asm/page.h>
+struct node_private;
+
/* Free memory management - zoned buddy allocator. */
#ifndef CONFIG_ARCH_FORCE_MAX_ORDER
#define MAX_PAGE_ORDER 10
@@ -1596,12 +1598,25 @@ typedef struct pglist_data {
atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
#ifdef CONFIG_NUMA
struct memory_tier __rcu *memtier;
+ struct node_private __rcu *node_private;
#endif
#ifdef CONFIG_MEMORY_FAILURE
struct memory_failure_stats mf_stats;
#endif
} pg_data_t;
+#ifdef CONFIG_NUMA
+static inline bool pgdat_is_private(pg_data_t *pgdat)
+{
+ return !!pgdat->node_private;
+}
+#else
+static inline bool pgdat_is_private(pg_data_t *pgdat)
+{
+ return false;
+}
+#endif
+
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
diff --git a/include/linux/node_private.h b/include/linux/node_private.h
new file mode 100644
index 0000000000000..475496c84249f
--- /dev/null
+++ b/include/linux/node_private.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_NODE_PRIVATE_H
+#define _LINUX_NODE_PRIVATE_H
+
+#include <linux/mm.h>
+#include <linux/nodemask.h>
+
+struct page;
+
+/**
+ * struct node_private - Per-node container for N_MEMORY_PRIVATE nodes
+ *
+ * Allocated by the driver and passed to node_private_register().
+ * The driver owns the memory and must ensure it remains valid until after
+ * node_private_unregister() returns.
+ *
+ * @owner: Opaque driver identifier
+ * @caps: NODE_PRIVATE_CAP_* service opt-ins for the node (zero by default;
+ * individual capabilities are defined and consumed by later changes)
+ */
+struct node_private {
+ void *owner;
+ unsigned long caps;
+};
+
+#ifdef CONFIG_NUMA
+#include <linux/mmzone.h>
+
+static inline bool folio_is_private_node(struct folio *folio)
+{
+ return node_state(folio_nid(folio), N_MEMORY_PRIVATE);
+}
+
+static inline bool page_is_private_node(struct page *page)
+{
+ return node_state(page_to_nid(page), N_MEMORY_PRIVATE);
+}
+
+static inline bool node_is_private(int nid)
+{
+ return node_state(nid, N_MEMORY_PRIVATE);
+}
+
+#else /* !CONFIG_NUMA */
+
+static inline bool folio_is_private_node(struct folio *folio)
+{
+ return false;
+}
+
+static inline bool page_is_private_node(struct page *page)
+{
+ return false;
+}
+
+static inline bool node_is_private(int nid)
+{
+ return false;
+}
+
+#endif /* CONFIG_NUMA */
+
+#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
+
+int node_private_register(int nid, struct node_private *np);
+int node_private_unregister(int nid);
+
+#else /* !CONFIG_NUMA || !CONFIG_MEMORY_HOTPLUG */
+
+static inline int node_private_register(int nid, struct node_private *np)
+{
+ return -ENODEV;
+}
+
+static inline int node_private_unregister(int nid)
+{
+ return 0;
+}
+
+#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
+
+#endif /* _LINUX_NODE_PRIVATE_H */
diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h
index b842aa5255464..ba3e6c570a112 100644
--- a/include/linux/nodemask.h
+++ b/include/linux/nodemask.h
@@ -391,6 +391,7 @@ enum node_states {
N_HIGH_MEMORY = N_NORMAL_MEMORY,
#endif
N_MEMORY, /* The node has memory(regular, high, movable)
*/
+ N_MEMORY_PRIVATE, /* The node's memory is private */
N_CPU, /* The node has one or more cpus */
N_GENERIC_INITIATOR, /* The node has one or more Generic Initiators
*/
NR_NODE_STATES
@@ -457,7 +458,7 @@ static __always_inline void node_set_offline(int nid)
static __always_inline int node_state(int node, enum node_states state)
{
- return node == 0;
+ return node == 0 && state != N_MEMORY_PRIVATE;
}
static __always_inline void node_set_state(int node, enum node_states state)
@@ -470,11 +471,11 @@ static __always_inline void node_clear_state(int node,
enum node_states state)
static __always_inline int num_node_state(enum node_states state)
{
- return 1;
+ return state == N_MEMORY_PRIVATE ? 0 : 1;
}
#define for_each_node_state(node, __state) \
- for ( (node) = 0; (node) == 0; (node) = 1)
+ for ((node) = 0; (node) == 0 && (__state) != N_MEMORY_PRIVATE; (node) =
1)
#define first_online_node 0
#define first_memory_node 0
diff --git a/mm/internal.h b/mm/internal.h
index 96d78a7778e88..cb9f4a8342e32 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -12,6 +12,7 @@
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/mmu_notifier.h>
+#include <linux/node_private.h>
#include <linux/pagemap.h>
#include <linux/pagewalk.h>
#include <linux/rmap.h>
@@ -98,6 +99,17 @@ extern int sysctl_min_unmapped_ratio;
extern int sysctl_min_slab_ratio;
#endif
+/* folio_is_private_managed() - folio has special mm management rules */
+static inline bool folio_is_private_managed(struct folio *folio)
+{
+ return folio_is_zone_device(folio) || folio_is_private_node(folio);
+}
+
+static inline bool page_is_private_managed(struct page *page)
+{
+ return folio_is_private_managed(page_folio(page));
+}
+
/*
* Maintains state across a page table move. The operation assumes both source
* and destination VMAs already exist and are specified by the user.
--
2.53.0-Meta
