From bc4a62b90a5361e22beb3f10190bcff224a8b7b0 Mon Sep 17 00:00:00 2001 From: Anthonin Bonnefoy Date: Tue, 20 Jan 2026 10:55:33 +0100 Subject: Auto-tune shared_buffers to use available huge pages With some environments, it's not possible to modify the amount of huge pages and only a fixed quantity is available. With this fixed amount, shared_buffers can be adjusted to try to fit and use as much huge pages as possible. However, this is very brittle as any modifying other parameters, like the amount of max_connections, will change the requested shared memory, and shared_buffers will need to be further adjusted. This patch introduces a new option to dynamically increase shared_buffers to use all available huge pages. --- src/backend/storage/buffer/buf_init.c | 51 +++++++++++++++++++ src/backend/utils/misc/guc_parameters.dat | 6 +++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/storage/bufmgr.h | 1 + src/include/storage/pg_shmem.h | 1 + 5 files changed, 60 insertions(+) diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index 6a57ab9cf30..8f3f3055c15 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -14,16 +14,19 @@ */ #include "postgres.h" +#include "utils/guc.h" #include "storage/aio.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" #include "storage/proclist.h" +#include "storage/pg_shmem.h" BufferDescPadded *BufferDescriptors; char *BufferBlocks; ConditionVariableMinimallyPadded *BufferIOCVArray; WritebackContext BackendWritebackContext; CkptSortItem *CkptBufferIds; +bool huge_pages_autotune_buffers = false; /* @@ -142,6 +145,54 @@ BufferManagerShmemInit(void) &backend_flush_after); } +/* + * BufferManagerAutotune + * + * auto-tune shared_buffers to use all remaining huge pages if + * huge_pages_autotune_buffers is enabled + */ +void +BufferManagerAutotune(Size requested_size) +{ + char buf[32]; + Size leftover_memory; + Size hugepagefree; + Size size_per_buffer; + Size additional_freelist_memory; + int candidate_nbuffers; + + if (!huge_pages_autotune_buffers) + /* No auto-tune requested */ + return; + + GetHugePageSize(NULL, &hugepagefree, NULL); + leftover_memory = hugepagefree - requested_size; + if (leftover_memory <= 0) + /* No leftover */ + return; + + size_per_buffer = sizeof(BufferDescPadded) + + sizeof(ConditionVariableMinimallyPadded) + + sizeof(CkptSortItem) + BLCKSZ; + candidate_nbuffers = NBuffers + leftover_memory / size_per_buffer; + + /* + * With the additional shared_buffers, the shared memory necessary for + * freelist-related structures will increase. We need to estimated this + * additional memory, and reduce the auto-tuned shared_buffers to fit in + * the available memory. + */ + additional_freelist_memory = StrategyShmemSize(candidate_nbuffers) - StrategyShmemSize(NBuffers); + candidate_nbuffers -= additional_freelist_memory / size_per_buffer; + + if (candidate_nbuffers <= 0) + return; + + snprintf(buf, sizeof(buf), "%d", candidate_nbuffers); + SetConfigOption("shared_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); +} + /* * BufferManagerShmemSize * diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 48590622b95..ff5ed89b307 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -1209,6 +1209,12 @@ options => 'huge_pages_options', }, +{ name => 'huge_pages_autotune_buffers', type => 'bool', context => 'PGC_POSTMASTER', group => 'RESOURCES_MEM', + short_desc => 'Autotune shared_buffers to use all free huge pages.', + variable => 'huge_pages_autotune_buffers', + boot_val => 'false', +}, + { name => 'huge_pages_status', type => 'enum', context => 'PGC_INTERNAL', group => 'PRESET_OPTIONS', short_desc => 'Indicates the status of huge pages.', flags => 'GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 256e8040092..ea600188841 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -133,6 +133,7 @@ # (change requires restart) #huge_pages = try # on, off, or try # (change requires restart) +#huge_pages_autotune_buffers = off # (change requires restart) #huge_page_size = 0 # zero for system default # (change requires restart) #temp_buffers = 8MB # min 800kB diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index a40adf6b2a8..09fe0258095 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -368,6 +368,7 @@ extern void MarkDirtyAllUnpinnedBuffers(int32 *buffers_dirtied, /* in buf_init.c */ extern void BufferManagerShmemInit(void); extern Size BufferManagerShmemSize(void); +extern void BufferManagerAutotune(Size requested_size); /* in localbuf.c */ extern void AtProcExit_LocalBuffers(void); diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h index 7b6efc9f306..4b1d61e6b33 100644 --- a/src/include/storage/pg_shmem.h +++ b/src/include/storage/pg_shmem.h @@ -46,6 +46,7 @@ extern PGDLLIMPORT int shared_memory_type; extern PGDLLIMPORT int huge_pages; extern PGDLLIMPORT int huge_page_size; extern PGDLLIMPORT int huge_pages_status; +extern PGDLLIMPORT bool huge_pages_autotune_buffers; /* Possible values for huge_pages and huge_pages_status */ typedef enum -- 2.52.0