On Mon, Jul 20, 2026 at 03:33:59PM +0800, Gregory Price wrote:
> N_MEMORY_PRIVATE nodes must be invisible to ordinary allocation,
> no allocations may reach one unless explicitly called for.
>
> Make this isolation structural in the zonelists: N_MEMORY_PRIVATE
> nodes are excluded from FALLBACK and NOFALLBACK entirely, making
> them effectively invisible to all normal callers of page_alloc.
>
> Add ZONELIST_PRIVATE, which spans (N_MEMORY | N_MEMORY_PRIVATE).
> When !CONFIG_NUMA, it aliases ZONELIST_FALLBACK and compiles out.
>
> Add ZONELIST_PRIVATE_NOFALLBACK as the __GFP_THISNODE target for
> private node allocations.
>
> Zonelist selection rides the allocator's alloc_flags. Add
> ALLOC_ZONELIST_PRIVATE and resolve it in select_zonelist(), which
> prepare_alloc_pages() applies from ac->alloc_flags; the default
> (ALLOC_DEFAULT) keeps node_zonelist()'s gfp-based selection.
>
> Add explicit private-node alloc() functions to prevent open-coding
> (both ride ALLOC_ZONELIST_PRIVATE through the alloc_flags-carrying
> entry points):
> - alloc_pages_node_private_noprof()
> - folio_alloc_node_private_noprof()
>
> alloc_contig_range adds an explicit node_is_private() guard because
> it dervices its target zones from PFNs rather than zonelists. All
> in-tree callers use N_MEMORY ranges, but the check prevents future
> callers from violating node isolation.
>
> Important note: By design this zonelist isolates N_MEMORY_PRIVATE
> from unintentional allocations, but does NOT isolate private-node
> allocations from falling back to non-private nodes.
>
> Signed-off-by: Gregory Price <[email protected]>
> ---
> include/linux/gfp.h | 11 ++++++++
> include/linux/mmzone.h | 11 +++++++-
> mm/mm_init.c | 2 +-
> mm/page_alloc.c | 64 ++++++++++++++++++++++++++++++++++++++++--
> mm/page_alloc.h | 26 +++++++++++++++++
> 5 files changed, 109 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/gfp.h b/include/linux/gfp.h
> index a327e58c313f8..f3e867de744f6 100644
> --- a/include/linux/gfp.h
> +++ b/include/linux/gfp.h
> @@ -204,6 +204,17 @@ static inline void arch_free_page(struct page *page, int
> order) { }
> static inline void arch_alloc_page(struct page *page, int order) { }
> #endif
>
> +/* Allocate from an N_MEMORY_PRIVATE node (selects the private zonelist). */
> +struct page *alloc_pages_node_private_noprof(gfp_t gfp, unsigned int order,
> + int nid);
> +#define alloc_pages_node_private(...) \
> + alloc_hooks(alloc_pages_node_private_noprof(__VA_ARGS__))
> +
> +struct folio *folio_alloc_node_private_noprof(gfp_t gfp, unsigned int order,
> + int nid);
> +#define folio_alloc_node_private(...) \
> + alloc_hooks(folio_alloc_node_private_noprof(__VA_ARGS__))
> +
> unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
> nodemask_t *nodemask, int nr_pages,
> struct page **page_array);
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 9815e48c03b97..654c5111f7cec 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -1392,13 +1392,22 @@ enum {
> #ifdef CONFIG_NUMA
> /*
> * The NUMA zonelists are doubled because we need zonelists that
> - * restrict the allocations to a single node for __GFP_THISNODE.
> + * restrict the allocations to a single node for __GFP_THISNODE
> + * and N_MEMORY_PRIVATE nodes (isolated from default lists).
> */
> ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */
> + ZONELIST_PRIVATE, /* N_MEMORY_PRIVATE access, falls back to DRAM
> */
> + ZONELIST_PRIVATE_NOFALLBACK, /* N_MEMORY_PRIVATE access, __GFP_THISNODE
> */
> #endif
> MAX_ZONELISTS
> };
>
> +#ifndef CONFIG_NUMA
> +/* Without NUMA only ZONELIST_FALLBACK exists so everything collapses there
> */
> +#define ZONELIST_PRIVATE ZONELIST_FALLBACK
> +#define ZONELIST_PRIVATE_NOFALLBACK ZONELIST_FALLBACK
> +#endif
> +
> /*
> * This struct contains information about a zone in a zonelist. It is stored
> * here to avoid dereferences into large structures and lookups of tables
> diff --git a/mm/mm_init.c b/mm/mm_init.c
> index 711f821f7b3c7..adb50647d29ca 100644
> --- a/mm/mm_init.c
> +++ b/mm/mm_init.c
> @@ -2701,7 +2701,7 @@ void __init mm_core_init(void)
> init_zero_page_pfn();
>
> /* Initializations relying on SMP setup */
> - BUILD_BUG_ON(MAX_ZONELISTS > 2);
> + BUILD_BUG_ON(MAX_ZONELISTS > 4);
> build_all_zonelists(NULL);
> page_alloc_init_cpuhp();
> alloc_tag_sec_init();
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index aa35bbe5d4c3e..78755a55b1540 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -5040,7 +5040,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask,
> unsigned int order,
> unsigned int *alloc_flags)
> {
> ac->highest_zoneidx = gfp_zone(gfp_mask);
> - ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
> + ac->zonelist = select_zonelist(preferred_nid, gfp_mask,
> ac->alloc_flags);
> ac->nodemask = nodemask;
> ac->migratetype = gfp_migratetype(gfp_mask);
>
> @@ -5346,7 +5346,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp,
> unsigned int order,
> unsigned int fastpath_alloc_flags = alloc_flags;
>
> /* Other flags could be supported later if needed. */
> - if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG)))
> + if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG |
> ALLOC_ZONELIST_PRIVATE)))
> return NULL;
>
> if (!alloc_order_allowed(gfp, order, alloc_flags))
> @@ -5457,6 +5457,22 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp,
> unsigned int order, int nid)
> }
> EXPORT_SYMBOL(__folio_alloc_node_noprof);
>
> +struct page *alloc_pages_node_private_noprof(gfp_t gfp, unsigned int order,
> + int nid)
> +{
> + return __alloc_pages_noprof(gfp, order, nid, NULL,
> + ALLOC_ZONELIST_PRIVATE);
> +}
> +EXPORT_SYMBOL_GPL(alloc_pages_node_private_noprof);
> +
> +struct folio *folio_alloc_node_private_noprof(gfp_t gfp, unsigned int order,
> + int nid)
> +{
> + return __folio_alloc_noprof(gfp, order, nid, NULL,
> + ALLOC_ZONELIST_PRIVATE);
> +}
> +EXPORT_SYMBOL_GPL(folio_alloc_node_private_noprof);
> +
> /*
> * Common helper functions. Never use with __GFP_HIGHMEM because the returned
> * address cannot represent highmem pages. Use alloc_pages and then kmap if
> @@ -5854,9 +5870,21 @@ static void build_zonelists_in_node_order(pg_data_t
> *pgdat, int *node_order,
> static void build_thisnode_zonelists(pg_data_t *pgdat)
> {
> struct zoneref *zonerefs;
> - int nr_zones;
> + int nr_zones = 0;
>
> + /*
> + * NOFALLBACK: the node's own zones. EMPTY for private nodes so a
> + * stray __GFP_THISNODE allocation can't violate isolation.
> + */
> zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
> + if (!node_is_private(pgdat->node_id))
> + nr_zones = build_zonerefs_node(pgdat, zonerefs);
> + zonerefs += nr_zones;
> + zonerefs->zone = NULL;
> + zonerefs->zone_idx = 0;
> +
> + /* PRIVATE_NOFALLBACK: the node's own zones, private nodes included. */
> + zonerefs = pgdat->node_zonelists[ZONELIST_PRIVATE_NOFALLBACK]._zonerefs;
> nr_zones = build_zonerefs_node(pgdat, zonerefs);
> zonerefs += nr_zones;
> zonerefs->zone = NULL;
> @@ -5899,11 +5927,28 @@ static void build_node_zonelist(pg_data_t *pgdat,
> const nodemask_t *candidates,
> static void build_zonelists(pg_data_t *pgdat)
> {
> static int node_order[MAX_NUMNODES];
> + nodemask_t tier_nodes;
> int local_node = pgdat->node_id;
> int node, nr_nodes = 0;
>
> memset(node_order, 0, sizeof(node_order));
>
> + /*
> + * Zonelists: FALLBACK, NOFALLBACK, PRIVATE
> + *
> + * FALLBACK: Allocation order for all nodes. Private nodes have lists
> + * but never appear as an entry in any list. Allocations
> + * targeting a private node w/ FALLBACK land on N_MEMORY.
> + *
> + * NOFALLBACK: A list for each node containing only itself.
> + * Allocations targeting a private node w/ NOFALLBACK
> + * will always fail (their NOFALLBACK is empty)
> + *
> + * PRIVATE: (N_MEMORY | N_MEMORY_PRIVATE) - allows access to
> + * private nodes, and falls back to normal memory unless
> + * __GFP_THISNODE otherwise constrains it.
> + */
> +
> build_node_zonelist(pgdat, &node_states[N_MEMORY], ZONELIST_FALLBACK,
> true, node_order, &nr_nodes);
> build_thisnode_zonelists(pgdat);
> @@ -5912,6 +5957,10 @@ static void build_zonelists(pg_data_t *pgdat)
> for (node = 0; node < nr_nodes; node++)
> pr_cont("%d ", node_order[node]);
> pr_cont("\n");
> +
> + nodes_or(tier_nodes, node_states[N_MEMORY],
> node_states[N_MEMORY_PRIVATE]);
Hi Gregory,
A question about this part, if my understanding is right,
ZONELIST_PRIVATE contains all private nodes, should we use some explicit target
nodemask here to make sure only the targeting owner's private node can be
touched?
Otherwise it might fall back into a different owner's private node.
--Richard
> + build_node_zonelist(pgdat, &tier_nodes, ZONELIST_PRIVATE, false,
> + node_order, &nr_nodes);
> }
>
> #ifdef CONFIG_HAVE_MEMORYLESS_NODES
> @@ -7282,6 +7331,15 @@ int alloc_contig_frozen_range_noprof(unsigned long
> start, unsigned long end,
> if (__alloc_contig_verify_gfp_mask(gfp_mask, (gfp_t *)&cc.gfp_mask))
> return -EINVAL;
>
> + /*
> + * Private nodes are kept unreachable via the zonelists, but
> + * alloc_contig works directly on a zone derived from the caller's
> + * pfn range, bypassing that. Reject this operation explicitly
> + * rather than silently carving memory out of an isolated node.
> + */
> + if (unlikely(node_is_private(zone_to_nid(cc.zone))))
> + return -EBUSY;
> +
> /*
> * What we do here is we mark all pageblocks in range as
> * MIGRATE_ISOLATE. Because pageblock and max order pages may
> diff --git a/mm/page_alloc.h b/mm/page_alloc.h
> index 23fc79ce97b60..741448d83ce77 100644
> --- a/mm/page_alloc.h
> +++ b/mm/page_alloc.h
> @@ -57,6 +57,9 @@
> */
> #define ALLOC_NO_CODETAG 0x1000
>
> +/* Select the N_MEMORY_PRIVATE zonelist family (see select_zonelist()). */
> +#define ALLOC_ZONELIST_PRIVATE 0x2000
> +
> /* Flags that allow allocations below the min watermark. */
> #define ALLOC_RESERVES
> (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
>
> @@ -95,6 +98,29 @@ struct alloc_context {
> unsigned int alloc_flags;
> };
>
> +#ifdef CONFIG_NUMA
> +static_assert(ZONELIST_PRIVATE + 1 == ZONELIST_PRIVATE_NOFALLBACK);
> +#endif
> +
> +/*
> + * select_zonelist - resolve the zonelist for an allocation
> + *
> + * The default matches node_zonelist(); ALLOC_ZONELIST_PRIVATE selects the
> + * private-node family so an allocation may reach an N_MEMORY_PRIVATE node.
> + */
> +static inline struct zonelist *
> +select_zonelist(int nid, gfp_t gfp, unsigned int alloc_flags)
> +{
> +#ifdef CONFIG_NUMA
> + if (alloc_flags & ALLOC_ZONELIST_PRIVATE) {
> + int idx = ZONELIST_PRIVATE + !!(gfp & __GFP_THISNODE);
> +
> + return &NODE_DATA(nid)->node_zonelists[idx];
> + }
> +#endif
> + return node_zonelist(nid, gfp);
> +}
> +
> /*
> * This function returns the order of a free page in the buddy system. In
> * general, page_zone(page)->lock must be held by the caller to prevent the
> --
> 2.53.0-Meta
>
>
