From 2ca3584a6baed6f2601f3dbd5c8734e2aa38d84f Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Fri, 13 Jan 2023 15:23:11 -0800 Subject: [PATCH v5 2/2] Add "table age" trigger concept to autovacuum. Teach autovacuum.c to launch "table age" autovacuums at the same point that it previously triggered antiwraparound autovacuums. Antiwraparound autovacuums are retained, but are only used as a true option of last resort, when regular autovacuum has presumably tried and failed to advance relfrozenxid (likely because the auto-cancel behavior kept cancelling regular autovacuums triggered based on table age). This is controlled via a new GUC, autovacuum_no_auto_cancel_age. The special auto-cancellation behavior applied by antiwraparound autovacuums is known to cause problems in the field, so it makes sense to avoid it, at least until the point where it starts to look like a proportionate response. Besides, the risk of the system eventually triggering xidStopLimit because of cancellations is a lot lower than it was back when the current auto-cancellation behavior was added by commit acac68b2. For example, there was no visibility map, so restarting antiwraparound autovacuum meant that the next autovacuum would get very little benefit from the work performed by earlier cancelled autovacuums. Author: Peter Geoghegan Reviewed-By: Andres Freund Reviewed-By: Jeff Davis Discussion: https://postgr.es/m/CAH2-Wz=S-R_2rO49Hm94Nuvhu9_twRGbTm6uwDRmRu-Sqn_t3w@mail.gmail.com --- src/include/postmaster/autovacuum.h | 1 + src/include/storage/proc.h | 2 +- src/backend/access/heap/visibilitymap.c | 5 +- src/backend/access/transam/multixact.c | 4 +- src/backend/commands/vacuum.c | 11 +- src/backend/postmaster/autovacuum.c | 119 +++++++++++++----- src/backend/storage/lmgr/proc.c | 4 +- src/backend/utils/misc/guc_tables.c | 16 ++- src/backend/utils/misc/postgresql.conf.sample | 1 + 9 files changed, 117 insertions(+), 46 deletions(-) diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index c140371b5..cb825f7e1 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -37,6 +37,7 @@ extern PGDLLIMPORT int autovacuum_vac_ins_thresh; extern PGDLLIMPORT double autovacuum_vac_ins_scale; extern PGDLLIMPORT int autovacuum_anl_thresh; extern PGDLLIMPORT double autovacuum_anl_scale; +extern PGDLLIMPORT double autovacuum_no_auto_cancel_age; extern PGDLLIMPORT int autovacuum_freeze_max_age; extern PGDLLIMPORT int autovacuum_multixact_freeze_max_age; extern PGDLLIMPORT double autovacuum_vac_cost_delay; diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index b5c6f46d0..8a92a9fe5 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -59,7 +59,7 @@ struct XidCache * CONCURRENTLY or REINDEX * CONCURRENTLY on non-expressional, * non-partial index */ -#define PROC_VACUUM_FOR_WRAPAROUND 0x08 /* set by autovac only */ +#define PROC_VACUUM_FOR_WRAPAROUND 0x08 /* emergency autovac */ #define PROC_IN_LOGICAL_DECODING 0x10 /* currently doing logical * decoding outside xact */ #define PROC_AFFECTS_ALL_HORIZONS 0x20 /* this proc's xmin must be diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 74ff01bb1..cbfb1822f 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -26,8 +26,9 @@ * per heap page. A set all-visible bit means that all tuples on the page are * known visible to all transactions, and therefore the page doesn't need to * be vacuumed. A set all-frozen bit means that all tuples on the page are - * completely frozen, and therefore the page doesn't need to be vacuumed even - * if whole table scanning vacuum is required (e.g. anti-wraparound vacuum). + * completely frozen. VACUUM doesn't give up the right to advance the rel's + * relfrozenxid/relminmxid just by skipping its all-frozen pages; it need only + * scan those pages that might have remaining unfrozen XIDs or MultiXactIds. * The all-frozen bit must be set only when the page is already all-visible. * * The map is conservative in the sense that we make sure that whenever a bit diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index e75e1fdf7..c0ee3876f 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -2553,8 +2553,8 @@ GetOldestMultiXactId(void) * info in MultiXactState, where it can be used to prevent overrun of old data * in the members SLRU area. * - * The return value is true if emergency autovacuum is required and false - * otherwise. + * The return value is true if emergency offset autovacuum (which appears as a + * table MXID age autovacuum to users) is required, and false otherwise. */ static bool SetOffsetVacuumLimit(bool is_startup) diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 18278acb5..dc63f1eed 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1054,8 +1054,8 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, /* * Determine the minimum freeze age to use: as specified by the caller, or * vacuum_freeze_min_age, but in any case not more than half - * autovacuum_freeze_max_age, so that autovacuums to prevent XID - * wraparound won't occur too frequently. + * autovacuum_freeze_max_age, so that table XID age autovacuums won't + * occur too frequently. */ if (freeze_min_age < 0) freeze_min_age = vacuum_freeze_min_age; @@ -1073,8 +1073,8 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, /* * Determine the minimum multixact freeze age to use: as specified by * caller, or vacuum_multixact_freeze_min_age, but in any case not more - * than half effective_multixact_freeze_max_age, so that autovacuums to - * prevent MultiXact wraparound won't occur too frequently. + * than half effective_multixact_freeze_max_age, so that table MXID age + * autovacuums won't occur too frequently. */ if (multixact_freeze_min_age < 0) multixact_freeze_min_age = vacuum_multixact_freeze_min_age; @@ -1863,7 +1863,8 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, bool skip_privs) * * We also set the VACUUM_FOR_WRAPAROUND flag, which is passed down by * autovacuum; it's used to avoid canceling a vacuum that was invoked - * in an emergency. + * because no earlier vacuum (in particular no earlier "table age" + * autovacuum) ran and advanced relfrozenxid/relminmxid. * * Note: these flags remain set until CommitTransaction or * AbortTransaction. We don't want to clear them until we reset diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 90f9df992..261649e82 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -123,6 +123,7 @@ int autovacuum_vac_ins_thresh; double autovacuum_vac_ins_scale; int autovacuum_anl_thresh; double autovacuum_anl_scale; +double autovacuum_no_auto_cancel_age; int autovacuum_freeze_max_age; int autovacuum_multixact_freeze_max_age; @@ -1150,8 +1151,8 @@ do_start_worker(void) { List *dblist; ListCell *cell; - TransactionId xidForceLimit; - MultiXactId multiForceLimit; + TransactionId xidAgeLimit; + MultiXactId multiAgeLimit; bool for_xid_wrap; bool for_multi_wrap; avw_dbase *avdb; @@ -1188,17 +1189,17 @@ do_start_worker(void) * particular tables, but not loosened.) */ recentXid = ReadNextTransactionId(); - xidForceLimit = recentXid - autovacuum_freeze_max_age; + xidAgeLimit = recentXid - autovacuum_freeze_max_age; /* ensure it's a "normal" XID, else TransactionIdPrecedes misbehaves */ /* this can cause the limit to go backwards by 3, but that's OK */ - if (xidForceLimit < FirstNormalTransactionId) - xidForceLimit -= FirstNormalTransactionId; + if (xidAgeLimit < FirstNormalTransactionId) + xidAgeLimit -= FirstNormalTransactionId; /* Also determine the oldest datminmxid we will consider. */ recentMulti = ReadNextMultiXactId(); - multiForceLimit = recentMulti - MultiXactMemberFreezeThreshold(); - if (multiForceLimit < FirstMultiXactId) - multiForceLimit -= FirstMultiXactId; + multiAgeLimit = recentMulti - MultiXactMemberFreezeThreshold(); + if (multiAgeLimit < FirstMultiXactId) + multiAgeLimit -= FirstMultiXactId; /* * Choose a database to connect to. We pick the database that was least @@ -1231,7 +1232,7 @@ do_start_worker(void) dlist_iter iter; /* Check to see if this one is at risk of wraparound */ - if (TransactionIdPrecedes(tmp->adw_frozenxid, xidForceLimit)) + if (TransactionIdPrecedes(tmp->adw_frozenxid, xidAgeLimit)) { if (avdb == NULL || TransactionIdPrecedes(tmp->adw_frozenxid, @@ -1242,7 +1243,7 @@ do_start_worker(void) } else if (for_xid_wrap) continue; /* ignore not-at-risk DBs */ - else if (MultiXactIdPrecedes(tmp->adw_minmulti, multiForceLimit)) + else if (MultiXactIdPrecedes(tmp->adw_minmulti, multiAgeLimit)) { if (avdb == NULL || MultiXactIdPrecedes(tmp->adw_minmulti, avdb->adw_minmulti)) @@ -1628,7 +1629,7 @@ AutoVacWorkerMain(int argc, char *argv[]) /* * Force synchronous replication off to allow regular maintenance even if * we are waiting for standbys to connect. This is important to ensure we - * aren't blocked from performing anti-wraparound tasks. + * aren't blocked from performing table age tasks. */ if (synchronous_commit > SYNCHRONOUS_COMMIT_LOCAL_FLUSH) SetConfigOption("synchronous_commit", "local", @@ -2942,8 +2943,8 @@ recheck_relation_needs_vacanalyze(Oid relid, * relation_needs_vacanalyze * * Check whether a relation needs to be vacuumed or analyzed; set each using - * "dovacuum" and "doanalyze", respectively. Also indicate if the vacuum is - * being forced because of Xid or multixact wraparound. + * "dovacuum" and "doanalyze", respectively. Also indicate whether the vacuum + * must use special antiwraparound protections by setting "wraparound". * * relopts is a pointer to the AutoVacOpts options (either for itself in the * case of a plain table, or for either itself or its parent table in the case @@ -2961,9 +2962,9 @@ recheck_relation_needs_vacanalyze(Oid relid, * the number of tuples (both live and dead) that there were as of the last * analyze. This is asymmetric to the VACUUM case. * - * We also force vacuum if the table's relfrozenxid is more than freeze_max_age - * transactions back, and if its relminmxid is more than - * multixact_freeze_max_age multixacts back. + * We also force table age vacuum if the table's relfrozenxid is more than + * freeze_max_age transactions back, and if its relminmxid is more than + * multixact_freeze_max_age multixacts back. This cannot be disabled. * * A table whose autovacuum_enabled option is false is * automatically skipped (unless we have to vacuum it due to freeze_max_age). @@ -3023,8 +3024,8 @@ relation_needs_vacanalyze(Oid relid, /* freeze parameters */ int freeze_max_age; int multixact_freeze_max_age; - TransactionId xidForceLimit; - MultiXactId multiForceLimit; + TransactionId xidAgeLimit; + MultiXactId multiAgeLimit; Assert(classForm != NULL); Assert(OidIsValid(relid)); @@ -3071,22 +3072,22 @@ relation_needs_vacanalyze(Oid relid, av_enabled = (relopts ? relopts->enabled : true); - /* Force vacuum if table is at risk of wraparound */ - xidForceLimit = recentXid - freeze_max_age; - if (xidForceLimit < FirstNormalTransactionId) - xidForceLimit -= FirstNormalTransactionId; - multiForceLimit = recentMulti - multixact_freeze_max_age; - if (multiForceLimit < FirstMultiXactId) - multiForceLimit -= FirstMultiXactId; + /* Force vacuum if table age exceeds cutoff */ + xidAgeLimit = recentXid - freeze_max_age; + if (xidAgeLimit < FirstNormalTransactionId) + xidAgeLimit -= FirstNormalTransactionId; + multiAgeLimit = recentMulti - multixact_freeze_max_age; + if (multiAgeLimit < FirstMultiXactId) + multiAgeLimit -= FirstMultiXactId; tableagevac = true; *wraparound = false; /* See header comments about trigger precedence */ if (TransactionIdIsNormal(relfrozenxid) && - TransactionIdPrecedes(relfrozenxid, xidForceLimit)) + TransactionIdPrecedes(relfrozenxid, xidAgeLimit)) trigger = AUTOVACUUM_TABLE_XID_AGE; else if (MultiXactIdIsValid(relminmxid) && - MultiXactIdPrecedes(relminmxid, multiForceLimit)) + MultiXactIdPrecedes(relminmxid, multiAgeLimit)) trigger = AUTOVACUUM_TABLE_MXID_AGE; else tableagevac = false; @@ -3099,13 +3100,69 @@ relation_needs_vacanalyze(Oid relid, return AUTOVACUUM_NONE; } - /* A table age autovacuum always gets antiwraparound protections */ - *wraparound = tableagevac; + /* + * If we're forcing table age autovacuum, are we at the point where it has + * to be an antiwraparound autovacuum? + * + * Antiwraparound autovacuums are different to other autovacuums in that + * they cannot be automatically canceled. They're used in emergencies, + * when no earlier standard table age autovacuum could complete and then + * advance the table's relfrozenxid/relminmxid, despite an ample table age + * autovacuum window. + */ + if (tableagevac) + { + /* + * Apply autovacuum_no_auto_cancel_age to determine the cutoff for + * antiwraparound no-auto-cancellation protection. This approach + * gives standard autovacuuming plenty of space to succeed, so we can + * be relatively confident that that hasn't and won't work out by the + * time antiwraparound mode finally starts to trigger. + * + * Positive values are interpreted as XID/MXID age values, whereas + * negative values are negated (made positive) for use as a multiplier + * on the freeze_max_age/multixact_freeze_max_age cutoffs. + */ + double no_cancel_xid_age = autovacuum_no_auto_cancel_age; + double no_cancel_mxid_age = autovacuum_no_auto_cancel_age; + int failsafe_age; + + if (autovacuum_no_auto_cancel_age < 0) + { + no_cancel_xid_age = + (double) freeze_max_age * -autovacuum_no_auto_cancel_age; + no_cancel_mxid_age = + (double) multixact_freeze_max_age * -autovacuum_no_auto_cancel_age; + } + + /* + * Never exceed failsafe age. Deliberately determine failsafe_age + * while using GUCs (not local variables) to clamp. This matches what + * vacuum_xid_failsafe_check does. + */ + failsafe_age = Max(vacuum_failsafe_age, + autovacuum_freeze_max_age * 1.05); + no_cancel_xid_age = Min(no_cancel_xid_age, (double) failsafe_age); + failsafe_age = Max(vacuum_multixact_failsafe_age, + autovacuum_multixact_freeze_max_age * 1.05); + no_cancel_mxid_age = Min(no_cancel_mxid_age, (double) failsafe_age); + + xidAgeLimit = recentXid - (int) no_cancel_xid_age; + if (xidAgeLimit < FirstNormalTransactionId) + xidAgeLimit -= FirstNormalTransactionId; + multiAgeLimit = recentMulti - (int) no_cancel_mxid_age; + if (multiAgeLimit < FirstMultiXactId) + multiAgeLimit -= FirstMultiXactId; + *wraparound = ((TransactionIdIsNormal(relfrozenxid) && + TransactionIdPrecedes(relfrozenxid, xidAgeLimit)) || + (MultiXactIdIsValid(relminmxid) && + MultiXactIdPrecedes(relminmxid, multiAgeLimit))); + } /* * If we found stats for the table, and autovacuum is currently enabled, * make a threshold-based decision whether to vacuum and/or analyze. If - * autovacuum is currently disabled, we must be here for anti-wraparound + * autovacuum is currently disabled, we must be here for forced table age * vacuuming only, so don't vacuum (or analyze) anything that's not being * forced. */ @@ -3160,7 +3217,7 @@ relation_needs_vacanalyze(Oid relid, { /* * Skip a table not found in stat hash, unless we have to force vacuum - * for anti-wrap purposes. If it's not acted upon, there's no need to + * for table age purposes. If it's not acted upon, there's no need to * vacuum it. */ *dovacuum = tableagevac; diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 00d26dc0f..dc66b9af0 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -1384,8 +1384,8 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) LWLockRelease(ProcArrayLock); /* - * Only do it if the worker is not working to protect against Xid - * wraparound. + * Only do it if the worker is not an antiwraparound autovacuum, a + * special type of autovacuum that is only used in emergencies */ if ((statusFlags & PROC_IS_AUTOVACUUM) && !(statusFlags & PROC_VACUUM_FOR_WRAPAROUND)) diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5025e80f8..02fcc112f 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3169,19 +3169,19 @@ struct config_int ConfigureNamesInt[] = { /* see varsup.c for why this is PGC_POSTMASTER not PGC_SIGHUP */ {"autovacuum_freeze_max_age", PGC_POSTMASTER, AUTOVACUUM, - gettext_noop("Age at which to autovacuum a table to prevent transaction ID wraparound."), + gettext_noop("Age at which to table XID age autovacuum a table."), NULL }, &autovacuum_freeze_max_age, - /* see vacuum_failsafe_age if you change the upper-limit value. */ + /* see vacuum_failsafe_age and autovacuum_no_auto_cancel_age if you change the upper-limit value. */ 200000000, 100000, 2000000000, NULL, NULL, NULL }, { /* see multixact.c for why this is PGC_POSTMASTER not PGC_SIGHUP */ {"autovacuum_multixact_freeze_max_age", PGC_POSTMASTER, AUTOVACUUM, - gettext_noop("Multixact age at which to autovacuum a table to prevent multixact wraparound."), + gettext_noop("Age at which to table MXID age autovacuum a table."), NULL }, &autovacuum_multixact_freeze_max_age, @@ -3706,6 +3706,16 @@ struct config_real ConfigureNamesReal[] = NULL, NULL, NULL }, + { + {"autovacuum_no_auto_cancel_age", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Age at which table age autovacuums apply antiwraparound no autocancellation protection."), + gettext_noop("Negative values are multipliers of autovacuum_freeze_max_age and autovacuum_multixact_freeze_max_age.") + }, + &autovacuum_no_auto_cancel_age, + -1.8, -100.0, 2000000000.0, + NULL, NULL, NULL + }, + { {"checkpoint_completion_target", PGC_SIGHUP, WAL_CHECKPOINTS, gettext_noop("Time spent flushing dirty buffers during checkpoint, as fraction of checkpoint interval."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 4cceda416..e21062b32 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -648,6 +648,7 @@ #autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table # size before insert vacuum #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_no_auto_cancel_age = -1.8 # controls forced vacuum autocancellation #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum # (change requires restart) #autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age -- 2.39.0