From cd92b6eb9651956bbce30efb78aa2fc923379448 Mon Sep 17 00:00:00 2001 From: Dinesh Salve Date: Sat, 14 Dec 2024 11:45:07 +0000 Subject: [PATCH] enable fetching explain plans from foreign server --- contrib/postgres_fdw/Makefile | 3 +- contrib/postgres_fdw/connection.c | 5 + .../postgres_fdw/expected/postgres_fdw.out | 92 +++++ contrib/postgres_fdw/option.c | 4 + contrib/postgres_fdw/postgres_fdw.c | 356 +++++++++++++++- contrib/postgres_fdw/postgres_fdw.h | 3 + .../postgres_fdw/postgres_fdw_auto_explain.c | 383 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 16 + src/backend/commands/explain.c | 56 +-- src/include/commands/explain.h | 2 + src/include/pg_config_ext.h | 8 + src/include/stamp-ext-h | 1 + 12 files changed, 889 insertions(+), 40 deletions(-) create mode 100644 contrib/postgres_fdw/postgres_fdw_auto_explain.c create mode 100644 src/include/pg_config_ext.h create mode 100644 src/include/stamp-ext-h diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile index 88fdce40d6..a94d04c61e 100644 --- a/contrib/postgres_fdw/Makefile +++ b/contrib/postgres_fdw/Makefile @@ -7,7 +7,8 @@ OBJS = \ deparse.o \ option.o \ postgres_fdw.o \ - shippable.o + shippable.o \ + postgres_fdw_auto_explain.o PGFILEDESC = "postgres_fdw - foreign data wrapper for PostgreSQL" PG_CPPFLAGS = -I$(libpq_srcdir) diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 2326f391d3..5ac51380ae 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -704,6 +704,11 @@ configure_remote_session(PGconn *conn) do_sql_command(conn, "SET extra_float_digits = 3"); else do_sql_command(conn, "SET extra_float_digits = 2"); + + if (get_postgres_fdw_show_remote_explain_enabled()) + { + do_sql_command(conn, "LOAD 'postgres_fdw'"); + } } /* diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index bf322198a2..d139d2e8e0 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -442,6 +442,98 @@ SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; fixed | (1 row) +-- run same queries with postgres_fdw.show_remote_explain_plans set on +SET postgres_fdw.show_remote_explain_plans = on; +-- single table without alias +EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c3, c1 OFFSET 100 LIMIT 10; + QUERY PLAN +----------------------------------- + Foreign Scan on ft1 + Remote Plan + Limit + -> Sort + Sort Key: c3, "C 1" + -> Seq Scan on "T 1" +(6 rows) + +-- single table with alias - also test that tableoid sort is not pushed to remote side +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c3, t1.c1, t1.tableoid OFFSET 100 LIMIT 10; + QUERY PLAN +------------------------------------------------------------------------------------- + Limit + Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid + -> Sort + Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid + Sort Key: t1.c3, t1.c1, t1.tableoid + -> Foreign Scan on public.ft1 t1 + Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" + Remote Plan + Seq Scan on "S 1"."T 1" + Output: "C 1", c2, c3, c4, c5, c6, c7, c8 +(11 rows) + +-- whole-row reference +EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.ft1 t1 + Output: t1.*, c3, c1 + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" ORDER BY c3 ASC NULLS LAST, "C 1" ASC NULLS LAST LIMIT 10::bigint OFFSET 100::bigint + Remote Plan + Limit + Output: "C 1", c2, c3, c4, c5, c6, c7, c8 + -> Sort + Output: "C 1", c2, c3, c4, c5, c6, c7, c8 + Sort Key: "T 1".c3, "T 1"."C 1" + -> Seq Scan on "S 1"."T 1" + Output: "C 1", c2, c3, c4, c5, c6, c7, c8 +(11 rows) + +-- with WHERE clause +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.ft1 t1 + Output: c1, c2, c3, c4, c5, c6, c7, c8 + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE ((c7 >= '1')) AND (("C 1" = 101)) AND ((c6 = '1')) + Remote Plan + Index Scan using t1_pkey on "S 1"."T 1" + Output: "C 1", c2, c3, c4, c5, c6, c7, c8 + Index Cond: ("T 1"."C 1" = 101) + Filter: (("T 1".c7 >= '1'::bpchar) AND (("T 1".c6)::text = '1'::text)) +(8 rows) + +-- with FOR UPDATE/SHARE +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = 101 FOR UPDATE; + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Foreign Scan on public.ft1 t1 + Output: c1, c2, c3, c4, c5, c6, c7, c8, t1.* + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE (("C 1" = 101)) FOR UPDATE + Remote Plan + LockRows + Output: "C 1", c2, c3, c4, c5, c6, c7, c8, ctid + -> Index Scan using t1_pkey on "S 1"."T 1" + Output: "C 1", c2, c3, c4, c5, c6, c7, c8, ctid + Index Cond: ("T 1"."C 1" = 101) +(9 rows) + +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = 102 FOR SHARE; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Foreign Scan on public.ft1 t1 + Output: c1, c2, c3, c4, c5, c6, c7, c8, t1.* + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE (("C 1" = 102)) FOR SHARE + Remote Plan + LockRows + Output: "C 1", c2, c3, c4, c5, c6, c7, c8, ctid + -> Index Scan using t1_pkey on "S 1"."T 1" + Output: "C 1", c2, c3, c4, c5, c6, c7, c8, ctid + Index Cond: ("T 1"."C 1" = 102) +(9 rows) + +SET postgres_fdw.show_remote_explain_plans = off; -- Test forcing the remote server to produce sorted data for a merge join. SET enable_hashjoin TO false; SET enable_nestloop TO false; diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c index 232d85354b..f5762eeb6e 100644 --- a/contrib/postgres_fdw/option.c +++ b/contrib/postgres_fdw/option.c @@ -586,4 +586,8 @@ _PG_init(void) NULL); MarkGUCPrefixReserved("postgres_fdw"); + + /* Setup hooks and gucs. */ + setup_postgres_fdw_gucs(); + setup_postgres_fdw_hooks(); } diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index c0810fbd7c..7ebf65df6e 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -59,6 +59,9 @@ PG_MODULE_MAGIC; /* If no remote estimates, assume a sort costs 20% extra */ #define DEFAULT_FDW_SORT_MULTIPLIER 1.2 +/* This is set implementation of post_parse_analyze_hook so that we know EXPLAIN command options. */ +extern ExplainState *pg_fdw_parsed_explain_state; + /* * Indexes of FDW-private information stored in fdw_private lists. * @@ -170,6 +173,7 @@ typedef struct PgFdwScanState MemoryContext temp_cxt; /* context for per-tuple temporary data */ int fetch_size; /* number of tuples per fetch */ + StringInfo remote_explain_plan; /* EXPLAIN plan received from foreign server */ } PgFdwScanState; /* @@ -541,7 +545,14 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo, const PgFdwRelationInfo *fpinfo_o, const PgFdwRelationInfo *fpinfo_i); static int get_batch_size_option(Relation rel); - +static void set_guc_boolean(PGconn *conn, char* guc, bool value); +static void set_guc_string(PGconn *conn, char* guc, char* value); +static void subscribe_postgres_fdw_notices(PGconn *conn, StringInfo explain_plans); +static void postgres_fdw_explain_notice_processor(void *arg, const char *notice); +bool static is_explain_query(const char *sql); +static void enrich_foreign_plans(char *sql, ExplainState *es, UserMapping *inputUser); +static UserMapping* get_remote_user(ForeignScanState *node); +static void append_foreign_explain_plan(ExplainState *es, StringInfo explainPlan); /* * Foreign-data wrapper handler function: return a struct with pointers @@ -1516,20 +1527,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) fsstate = (PgFdwScanState *) palloc0(sizeof(PgFdwScanState)); node->fdw_state = fsstate; - /* - * Identify which user to do the remote access as. This should match what - * ExecCheckPermissions() does. - */ - userid = OidIsValid(fsplan->checkAsUser) ? fsplan->checkAsUser : GetUserId(); - if (fsplan->scan.scanrelid > 0) - rtindex = fsplan->scan.scanrelid; - else - rtindex = bms_next_member(fsplan->fs_base_relids, -1); - rte = exec_rt_fetch(rtindex, estate); - - /* Get info about foreign table. */ - table = GetForeignTable(rte->relid); - user = GetUserMapping(userid, table->serverid); + user = get_remote_user(node); /* * Get connection to the foreign server. Connection manager will @@ -1537,6 +1535,37 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) */ fsstate->conn = GetConnection(user, false, &fsstate->conn_state); + /* + * If this is EXPLAIN command, set guc on remote connection to provide options to foreign server for plan generation. + */ + if (is_explain_query(estate->es_sourceText) && get_postgres_fdw_show_remote_explain_enabled()) { + set_guc_boolean(fsstate->conn, "postgres_fdw.auto_explain_enabled", true); + set_guc_boolean(fsstate->conn, "postgres_fdw.analyze_enabled", pg_fdw_parsed_explain_state->analyze); + + char *expected_explain_format; + switch (pg_fdw_parsed_explain_state->format) + { + case EXPLAIN_FORMAT_TEXT: + expected_explain_format = "text"; + break; + case EXPLAIN_FORMAT_JSON: + expected_explain_format = "json"; + break; + case EXPLAIN_FORMAT_YAML: + expected_explain_format = "yaml"; + break; + case EXPLAIN_FORMAT_XML: + expected_explain_format = "xml"; + break; + } + set_guc_string(fsstate->conn, "postgres_fdw.format_enabled", expected_explain_format); + set_guc_boolean(fsstate->conn, "postgres_fdw.settings_enabled", pg_fdw_parsed_explain_state->settings); + set_guc_boolean(fsstate->conn, "postgres_fdw.verbose_enabled", pg_fdw_parsed_explain_state->verbose); + set_guc_boolean(fsstate->conn, "postgres_fdw.buffers_enabled", pg_fdw_parsed_explain_state->buffers); + set_guc_boolean(fsstate->conn, "postgres_fdw.wal_enabled", pg_fdw_parsed_explain_state->wal); + set_guc_boolean(fsstate->conn, "postgres_fdw.timing_enabled", pg_fdw_parsed_explain_state->timing); + } + /* Assign a unique ID for my cursor */ fsstate->cursor_number = GetCursorNumber(fsstate->conn); fsstate->cursor_exists = false; @@ -1574,6 +1603,14 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) fsstate->attinmeta = TupleDescGetAttInMetadata(fsstate->tupdesc); + /* + * Subscribe to NOTICES received from foreign server. + */ + if (get_postgres_fdw_show_remote_explain_enabled() && is_explain_query(estate->es_sourceText)) { + fsstate->remote_explain_plan = makeStringInfo(); + subscribe_postgres_fdw_notices(fsstate->conn, fsstate->remote_explain_plan); + } + /* * Prepare for processing of parameters used in remote query, if any. */ @@ -2822,6 +2859,114 @@ postgresEndDirectModify(ForeignScanState *node) /* MemoryContext will be deleted automatically. */ } +/* + * Append plan received from foreign server to ExplainState. + */ +static void +append_foreign_explain_plan(ExplainState *es, StringInfo explainPlan) +{ + if (explainPlan->len) + { + /* Append "Remote Plan" header */ + switch (es->format) + { + case EXPLAIN_FORMAT_TEXT: + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfo(es->str, "Remote Plan\n"); + break; + case EXPLAIN_FORMAT_JSON: + appendStringInfo(es->str, ",\n"); + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfo(es->str, "\"Remote Plan\": "); + break; + case EXPLAIN_FORMAT_XML: + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfo(es->str, "\n"); + break; + case EXPLAIN_FORMAT_YAML: + appendStringInfo(es->str, "\n"); + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfo(es->str, "Remote-Plan:\n"); + break; + } + + /* Append fetched remote plan */ + char *explainPlans = explainPlan->data; + if (explainPlans) { + // remove extra newlines at the end + char *last = explainPlans + explainPlan->len; + while (last > explainPlans && (*last == '\n' || *last == '\0')) + { + *last = '\0'; + last--; + } + } + + /* Fetch explain plan from { */ + // why formatting changes are required + if (explainPlans && es->format == EXPLAIN_FORMAT_JSON) + { + // explainPlans = strchr(explainPlans, '{'); + } + else if (es->format == EXPLAIN_FORMAT_YAML) { + explainPlans = strchr(explainPlans, 'P'); + } + bool firstLine = true; + while (explainPlans && strlen(explainPlans) > 0) + { + // this is last line in JSON format + if (es->format == EXPLAIN_FORMAT_JSON && strcmp(explainPlans, "]") == 0) { + // appendStringInfoSpaces(es->str, es->indent * 2); + // appendStringInfoString(es->str,"}"); + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfoString(es->str,"]"); + break; + } + + /* add extra indent if not first line in json */ + if (es->format == EXPLAIN_FORMAT_JSON && firstLine) { + firstLine = false; + } + else if (es->format == EXPLAIN_FORMAT_JSON) { + appendStringInfoSpaces(es->str, es->indent * 2); + } + else if (es->format == EXPLAIN_FORMAT_YAML) { + appendStringInfoSpaces(es->str, es->indent * 2 + 4); + } + else if (es->format == EXPLAIN_FORMAT_XML) { + appendStringInfoSpaces(es->str, es->indent * 2); + } + else { + appendStringInfoSpaces(es->str, es->indent * 2 + 2); + } + + /* Temporarily replace earliest '\n' with '\0' to get current line */ + char *curLine = strchr(explainPlans, '\n'); + if (curLine) + *curLine = '\0'; + + if (curLine) /* if newline in curline, add '\n' at end */ + appendStringInfo(es->str,"%s\n", explainPlans); + else + appendStringInfo(es->str,"%s", explainPlans); + + /* Restore '\n' */ + if (curLine) + *curLine = '\n'; + + explainPlans = curLine ? (curLine+1) : NULL; + } + + /* Append remote plan footer */ + switch (es->format) + { + case EXPLAIN_FORMAT_XML: + appendStringInfo(es->str, "\n"); + break; + } + } +} + /* * postgresExplainForeignScan * Produce extra output for EXPLAIN of a ForeignScan on a foreign table @@ -2831,6 +2976,7 @@ postgresExplainForeignScan(ForeignScanState *node, ExplainState *es) { ForeignScan *plan = castNode(ForeignScan, node->ss.ps.plan); List *fdw_private = plan->fdw_private; + EState *estate = node->ss.ps.state; /* * Identify foreign scans that are really joins or upper relations. The @@ -2927,6 +3073,28 @@ postgresExplainForeignScan(ForeignScanState *node, ExplainState *es) sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql)); ExplainPropertyText("Remote SQL", sql, es); } + + if (get_postgres_fdw_show_remote_explain_enabled()) + { + /* For analyze = false, we explicitly fetch query plans by executing EXPLAIN on remote shard. */ + if (!es->analyze) + { + int rtindex; + char *sql; + sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql)); + UserMapping *user = get_remote_user(node); + enrich_foreign_plans(sql, es, user); + } + else if (is_explain_query(node->ss.ps.state->es_sourceText)) { + PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state; + if (fsstate->remote_explain_plan->len > 0) { + append_foreign_explain_plan(es, fsstate->remote_explain_plan); + if (es->format == EXPLAIN_FORMAT_TEXT) { + appendStringInfo(es->str, "\n"); + } + } + } + } } /* @@ -3804,8 +3972,15 @@ static void fetch_more_data(ForeignScanState *node) { PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state; + ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan; + EState *estate = node->ss.ps.state; PGresult *volatile res = NULL; MemoryContext oldcontext; + RangeTblEntry *rte; + Oid userid; + ForeignTable *table; + UserMapping *user; + int rtindex; /* * We'll store the tuples in the batch_cxt. First, flush the previous @@ -7963,3 +8138,154 @@ get_batch_size_option(Relation rel) return batch_size; } + +/* + * Identify which user to do the remote access as. This should match what + * ExecCheckPermissions() does. + */ +static UserMapping* +get_remote_user(ForeignScanState *node) +{ + ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan; + EState *estate = node->ss.ps.state; + Oid userid; + int rtindex; + RangeTblEntry *rte; + ForeignTable *table; + + userid = OidIsValid(fsplan->checkAsUser) ? fsplan->checkAsUser : GetUserId(); + if (fsplan->scan.scanrelid > 0) + rtindex = fsplan->scan.scanrelid; + else + rtindex = bms_next_member(fsplan->fs_base_relids, -1); + rte = exec_rt_fetch(rtindex, estate); + + /* Get info about foreign table. */ + table = GetForeignTable(rte->relid); + return GetUserMapping(userid, table->serverid); +} + +/* + * Connect to remote shards and retreive the explain plans for the given sql. + */ +static void +enrich_foreign_plans(char *sql, ExplainState *es, UserMapping *inputUser) { + PGresult *volatile res = NULL; + PGconn *conn; + StringInfoData sqlE, multiLineplans; + + /* Prepare EXPLAIN command to be sent to foreign server. */ + initStringInfo(&sqlE); + appendStringInfoString(&sqlE, "EXPLAIN ("); + appendStringInfo(&sqlE, "ANALYZE %s", es->analyze? "true" : "false"); + appendStringInfo(&sqlE, ", VERBOSE %s", es->verbose? "true" : "false"); + appendStringInfo(&sqlE, ", COSTS %s", es->costs? "true" : "false"); + appendStringInfo(&sqlE, ", SETTINGS %s", es->settings? "true" : "false"); + appendStringInfo(&sqlE, ", BUFFERS %s", es->buffers? "true" : "false"); + + if (es->serialize == EXPLAIN_SERIALIZE_NONE) { + appendStringInfoString(&sqlE, ", SERIALIZE OFF"); + } + else if (es->serialize == EXPLAIN_SERIALIZE_TEXT) { + appendStringInfoString(&sqlE, ", SERIALIZE TEXT"); + } + else if (es->serialize == EXPLAIN_SERIALIZE_BINARY) { + appendStringInfoString(&sqlE, ", SERIALIZE BINARY"); + } + + appendStringInfo(&sqlE, ", WAL %s", es->wal? "true" : "false"); + appendStringInfo(&sqlE, ", TIMING %s", es->timing? "true" : "false"); + appendStringInfo(&sqlE, ", SUMMARY %s", es->summary? "true" : "false"); + appendStringInfo(&sqlE, ", MEMORY %s", es->memory? "true" : "false"); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + appendStringInfoString(&sqlE, ", FORMAT TEXT"); + } + else if (es->format == EXPLAIN_FORMAT_JSON) + { + appendStringInfoString(&sqlE, ", FORMAT JSON"); + } + else if (es->format == EXPLAIN_FORMAT_XML) + { + appendStringInfoString(&sqlE, ", FORMAT XML"); + } + else if (es->format == EXPLAIN_FORMAT_YAML) + { + appendStringInfoString(&sqlE, ", FORMAT YAML"); + } + appendStringInfoString(&sqlE, ")"); + appendStringInfoString(&sqlE, sql); + + conn = GetConnection(inputUser, false, NULL); + + PG_TRY(); + { + // Run the query and collect the remote plan + res = pgfdw_exec_query(conn, sqlE.data, NULL); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pgfdw_report_error(ERROR, res, conn, false, sql); + int numrows = PQntuples(res); + + initStringInfo(&multiLineplans); + for (int i = 0; i < numrows; i++) { + appendStringInfoString(&multiLineplans, PQgetvalue(res, i, 0)); + if (i != numrows - 1) + appendStringInfoString(&multiLineplans, "\n"); + } + append_foreign_explain_plan(es, &multiLineplans); + } + PG_FINALLY(); + { + if (res) + PQclear(res); + } + PG_END_TRY(); + + ReleaseConnection(conn); +} + +static void set_guc_boolean(PGconn *conn, char* guc, bool value) +{ + int guc_sql_len = ((strlen("SET LOCAL = ") + strlen(guc)) * sizeof(char)) + sizeof(int); + char* guc_sql = palloc0(guc_sql_len + 1); + snprintf(guc_sql, guc_sql_len, "SET LOCAL %s = %d", guc, value); + do_sql_command(conn, guc_sql); +} + +static void set_guc_string(PGconn *conn, char* guc, char* value) +{ + int guc_sql_len = ((strlen("SET LOCAL = ") + strlen(guc) + strlen(value)) * sizeof(char)) + sizeof(int); + char* guc_sql = palloc0(guc_sql_len + 1); + snprintf(guc_sql, guc_sql_len, "SET LOCAL %s = %s", guc, value); + do_sql_command(conn, guc_sql); +} + +/* + * This is callback function for NOTICEs from remote shards for EXPLAIN ANALYZE queries. + */ +static void postgres_fdw_explain_notice_processor(void *arg, const char *notice) { + StringInfo explain_plans = (char **) arg; + // We might receive plans per batch of cursor, but we only need to store one. + // do we really need to handle len==0. report warn if we still recived. have test around this warn. + if (strstr(notice, "postgres_fdw_explain_plan") && explain_plans->len == 0) { + char *explain_plan_str = strchr(strchr(notice, ':') + 1, ':') + 1; + appendStringInfoString(explain_plans, explain_plan_str); + } +} + +/* + * Remote shards sends the EXPLAIN PLANS as a NOTICE to the host shard when a guc is set for EXPLAIN ANALYZE queries. + * To listen to those NOTICEs, here we subscribe to NOTICEs on this connection and register callback so that + * callback function is called instead of defaultNoticeProcessor. + */ +static void subscribe_postgres_fdw_notices(PGconn *conn, StringInfo explain_plans) { + PQsetNoticeProcessor(conn, postgres_fdw_explain_notice_processor, explain_plans); +} + +/* + * Return true if this is EXPLAIN query. + */ +bool static is_explain_query(const char *sql) { + return pg_strncasecmp("EXPLAIN", sql, 7) == 0; +} \ No newline at end of file diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h index 9e501660d1..7c105ade83 100644 --- a/contrib/postgres_fdw/postgres_fdw.h +++ b/contrib/postgres_fdw/postgres_fdw.h @@ -257,4 +257,7 @@ extern const char *get_jointype_name(JoinType jointype); extern bool is_builtin(Oid objectId); extern bool is_shippable(Oid objectId, Oid classId, PgFdwRelationInfo *fpinfo); +extern void setup_postgres_fdw_hooks(void); +extern void setup_postgres_fdw_gucs(void); + #endif /* POSTGRES_FDW_H */ diff --git a/contrib/postgres_fdw/postgres_fdw_auto_explain.c b/contrib/postgres_fdw/postgres_fdw_auto_explain.c new file mode 100644 index 0000000000..699792e031 --- /dev/null +++ b/contrib/postgres_fdw/postgres_fdw_auto_explain.c @@ -0,0 +1,383 @@ +/*------------------------------------------------------------------------- + * + * postgres_fdw_auto_explain.c + * + * + * Copyright (c) 2008-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/postgres_fdw/postgres_fdw_auto_explain.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "access/parallel.h" +#include "commands/explain.h" +#include "common/pg_prng.h" +#include "executor/instrument.h" +#include "utils/guc.h" +#include "parser/analyze.h" + +/* GUC variables */ +static bool postgres_fdw_show_remote_explain_plans = false; +static bool postgres_fdw_auto_explain_enabled = false; +static bool postgres_fdw_analyze = false; +static bool postgres_fdw_verbose = false; +static bool postgres_fdw_buffers = false; +static bool postgres_fdw_wal = false; +static bool postgres_fdw_triggers = false; +static bool postgres_fdw_timing = true; +static bool postgres_fdw_settings = false; +static int postgres_fdw_format = EXPLAIN_FORMAT_TEXT; + + +static const struct config_enum_entry format_options[] = { + {"text", EXPLAIN_FORMAT_TEXT, false}, + {"xml", EXPLAIN_FORMAT_XML, false}, + {"json", EXPLAIN_FORMAT_JSON, false}, + {"yaml", EXPLAIN_FORMAT_YAML, false}, + {NULL, 0, false} +}; + +/* Current nesting depth of ExecutorRun calls */ +static int nesting_level = 0; + +#define auto_explain_enabled() \ + (postgres_fdw_auto_explain_enabled && nesting_level == 1) + + +/* Saved hook values in case of unload */ +static ExecutorStart_hook_type prev_ExecutorStart = NULL; +static ExecutorRun_hook_type prev_ExecutorRun = NULL; +static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; +static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; +static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; + +ExplainState *pg_fdw_parsed_explain_state; +static bool pg_fdw_load_initialized = false; + +static void explain_ExecutorStart(QueryDesc *queryDesc, int eflags); +static void explain_ExecutorRun(QueryDesc *queryDesc, + ScanDirection direction, + uint64 count, bool execute_once); +static void explain_ExecutorFinish(QueryDesc *queryDesc); +static void explain_ExecutorEnd(QueryDesc *queryDesc); +static void postgres_fdw_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate); +bool get_postgres_fdw_show_remote_explain_enabled(); + +void setup_postgres_fdw_hooks() +{ + prev_ExecutorStart = ExecutorStart_hook; + ExecutorStart_hook = explain_ExecutorStart; + prev_ExecutorRun = ExecutorRun_hook; + ExecutorRun_hook = explain_ExecutorRun; + prev_ExecutorFinish = ExecutorFinish_hook; + ExecutorFinish_hook = explain_ExecutorFinish; + prev_ExecutorEnd = ExecutorEnd_hook; + ExecutorEnd_hook = explain_ExecutorEnd; + + prev_post_parse_analyze_hook = post_parse_analyze_hook; + post_parse_analyze_hook = postgres_fdw_post_parse_analyze; + pg_fdw_load_initialized = true; +} + +void setup_postgres_fdw_gucs() +{ + DefineCustomBoolVariable("postgres_fdw.show_remote_explain_plans", + "If enabled, plan from foreign server is embeded in EXPLAIN command output", + NULL, + &postgres_fdw_show_remote_explain_plans, + false, + PGC_SIGHUP | PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.auto_explain_enabled", + "If enabled, explain plan is sent as a NOTICE", + NULL, + &postgres_fdw_auto_explain_enabled, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.analyze_enabled", + "Use EXPLAIN ANALYZE for plan logging.", + NULL, + &postgres_fdw_analyze, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.settings_enabled", + "Log modified configuration parameters affecting query planning.", + NULL, + &postgres_fdw_settings, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.verbose_enabled", + "Use EXPLAIN VERBOSE for plan logging.", + NULL, + &postgres_fdw_verbose, + false, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.buffers_enabled", + "Log buffers usage.", + NULL, + &postgres_fdw_buffers, + false, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.wal_enabled", + "Log WAL usage.", + NULL, + &postgres_fdw_wal, + false, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.triggers_enabled", + "Include trigger statistics in plans.", + "This has no effect unless analyze is also set.", + &postgres_fdw_triggers, + false, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomEnumVariable("postgres_fdw.format_enabled", + "EXPLAIN format to be used for plan logging.", + NULL, + &postgres_fdw_format, + EXPLAIN_FORMAT_TEXT, + format_options, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable("postgres_fdw.timing_enabled", + "Collect timing data, not just row counts.", + NULL, + &postgres_fdw_timing, + true, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); +} + +static void +postgres_fdw_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate) +{ + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query, jstate); + + if (query->utilityStmt != NULL && IsA(query->utilityStmt, ExplainStmt)) { + pg_fdw_parsed_explain_state = NULL; + ExplainStmt *stmt = (ExplainStmt *) query->utilityStmt; + pg_fdw_parsed_explain_state = ParseExplainStmtOptions(pstate, stmt->options); + } +} + +/* + * ExecutorStart hook: start up logging if needed + */ +static void +explain_ExecutorStart(QueryDesc *queryDesc, int eflags) +{ + if (auto_explain_enabled()) + { + /* Enable per-node instrumentation iff analyze is required. */ + if (postgres_fdw_analyze && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + { + if (postgres_fdw_timing) + queryDesc->instrument_options |= INSTRUMENT_TIMER; + else + queryDesc->instrument_options |= INSTRUMENT_ROWS; + if (postgres_fdw_buffers) + queryDesc->instrument_options |= INSTRUMENT_BUFFERS; + if (postgres_fdw_wal) + queryDesc->instrument_options |= INSTRUMENT_WAL; + } + } + + if (prev_ExecutorStart) + prev_ExecutorStart(queryDesc, eflags); + else + standard_ExecutorStart(queryDesc, eflags); + + if (auto_explain_enabled()) + { + /* + * Set up to track total elapsed time in ExecutorRun. Make sure the + * space is allocated in the per-query context so it will go away at + * ExecutorEnd. + */ + if (queryDesc->totaltime == NULL) + { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(queryDesc->estate->es_query_cxt); + queryDesc->totaltime = InstrAlloc(1, INSTRUMENT_ALL, false); + MemoryContextSwitchTo(oldcxt); + } + } +} + +/* + * ExecutorRun hook: all we need do is track nesting depth + */ +static void +explain_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, + uint64 count, bool execute_once) +{ + nesting_level++; + PG_TRY(); + { + if (prev_ExecutorRun) + prev_ExecutorRun(queryDesc, direction, count); + else + standard_ExecutorRun(queryDesc, direction, count); + + if (auto_explain_enabled()) { + // send explain plans + ExplainState *es = NewExplainState(); + + es->analyze = (queryDesc->instrument_options && postgres_fdw_analyze); + es->verbose = postgres_fdw_verbose; + es->buffers = (es->analyze && postgres_fdw_buffers); + es->wal = (es->analyze && postgres_fdw_wal); + es->timing = (es->analyze && postgres_fdw_timing); + es->summary = es->analyze; + /* No support for MEMORY */ + /* es->memory = false; */ + es->format = postgres_fdw_format; + es->settings = postgres_fdw_settings; + + ExplainBeginOutput(es); + ExplainQueryParameters(es, queryDesc->params, log_parameter_max_length); + ExplainPrintPlan(es, queryDesc); + if (es->analyze && postgres_fdw_triggers) + ExplainPrintTriggers(es, queryDesc); + if (es->costs) + ExplainPrintJITSummary(es, queryDesc); + ExplainEndOutput(es); + + /* Remove last line break */ + if (es->str->len > 0 && es->str->data[es->str->len - 1] == '\n') + es->str->data[--es->str->len] = '\0'; + + /* Fix JSON to output an object */ + if (postgres_fdw_format == EXPLAIN_FORMAT_JSON) + { + es->str->data[0] = '{'; + es->str->data[es->str->len - 1] = '}'; + } + + /* + * Note: we rely on the existing logging of context or + * debug_query_string to identify just which statement is being + * reported. This isn't ideal but trying to do it here would + * often result in duplication. + */ + ereport(LOG, + (errmsg("postgres_fdw_auto_explain_enabled:%d", postgres_fdw_auto_explain_enabled), + errhidestmt(true))); + ereport(LOG, + (errmsg("sending explain plan to caller:%s", es->str->data), + errhidestmt(true))); + // receive token from source server and add that in response so that no other code can impact this. + ereport(NOTICE, + (errmsg("postgres_fdw_explain_plan:%s", es->str->data), + errhidestmt(true))); + } + } + PG_FINALLY(); + { + nesting_level--; + } + PG_END_TRY(); +} + +/* + * ExecutorFinish hook: all we need do is track nesting depth + */ +static void +explain_ExecutorFinish(QueryDesc *queryDesc) +{ + nesting_level++; + PG_TRY(); + { + if (prev_ExecutorFinish) + prev_ExecutorFinish(queryDesc); + else + standard_ExecutorFinish(queryDesc); + } + PG_FINALLY(); + { + nesting_level--; + } + PG_END_TRY(); +} + +/* + * ExecutorEnd hook: log results if needed + */ +static void +explain_ExecutorEnd(QueryDesc *queryDesc) +{ + if (prev_ExecutorEnd) + prev_ExecutorEnd(queryDesc); + else + standard_ExecutorEnd(queryDesc); +} + +bool get_postgres_fdw_show_remote_explain_enabled() +{ + if (postgres_fdw_show_remote_explain_plans) { + /* + * Check if extension is loaded and we have required hooks initialized. This happens when postgres_fdw wasn't + * already loaded and this is first SQL who need to access foreign server. + */ + if (!pg_fdw_load_initialized) { + ereport(ERROR, + (errmsg("postgres_fdw.show_remote_explain_plans is set but extension postgres_fdw is not loaded yet."), + errhidestmt(true))); + return false; + } + return true; + } + return false; +} \ No newline at end of file diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 3900522ccb..caf4264f0d 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -285,6 +285,22 @@ SELECT * FROM ft1 t1 WHERE t1.c3 = (SELECT MAX(c3) FROM ft2 t2) ORDER BY c1; WITH t1 AS (SELECT * FROM ft1 WHERE c1 <= 10) SELECT t2.c1, t2.c2, t2.c3, t2.c4 FROM t1, ft2 t2 WHERE t1.c1 = t2.c1 ORDER BY t1.c1; -- fixed values SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; + +-- run same queries with postgres_fdw.show_remote_explain_plans set on +SET postgres_fdw.show_remote_explain_plans = on; +-- single table without alias +EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c3, c1 OFFSET 100 LIMIT 10; +-- single table with alias - also test that tableoid sort is not pushed to remote side +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c3, t1.c1, t1.tableoid OFFSET 100 LIMIT 10; +-- whole-row reference +EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10; +-- with WHERE clause +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; +-- with FOR UPDATE/SHARE +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = 101 FOR UPDATE; +EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = 102 FOR SHARE; +SET postgres_fdw.show_remote_explain_plans = off; + -- Test forcing the remote server to produce sorted data for a merge join. SET enable_hashjoin TO false; SET enable_nestloop TO false; diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index a201ed3082..d3b2386572 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -181,28 +181,20 @@ static void ExplainYAMLLineStarting(ExplainState *es); static void escape_yaml(StringInfo buf, const char *str); static SerializeMetrics GetSerializationMetrics(DestReceiver *dest); - - -/* - * ExplainQuery - - * execute an EXPLAIN command - */ -void -ExplainQuery(ParseState *pstate, ExplainStmt *stmt, - ParamListInfo params, DestReceiver *dest) +ExplainState * +ParseExplainStmtOptions(ParseState *pstate, List *options) { - ExplainState *es = NewExplainState(); - TupOutputState *tstate; - JumbleState *jstate = NULL; - Query *query; - List *rewritten; ListCell *lc; bool timing_set = false; bool buffers_set = false; bool summary_set = false; + ExplainState *es = NewExplainState(); + + if (!options) + return es; /* Parse options list. */ - foreach(lc, stmt->options) + foreach(lc, options) { DefElem *opt = (DefElem *) lfirst(lc); @@ -287,18 +279,37 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, parser_errposition(pstate, opt->location))); } + /* if the timing was not set explicitly, set default value */ + es->timing = (timing_set) ? es->timing : es->analyze; + + /* if the summary was not set explicitly, set default value */ + es->summary = (summary_set) ? es->summary : es->analyze; + + return es; +} + +/* + * ExplainQuery - + * execute an EXPLAIN command + */ +void +ExplainQuery(ParseState *pstate, ExplainStmt *stmt, + ParamListInfo params, DestReceiver *dest) +{ + ExplainState *es; + TupOutputState *tstate; + JumbleState *jstate = NULL; + Query *query; + List *rewritten; + + es = ParseExplainStmtOptions(pstate, stmt->options); + /* check that WAL is used with EXPLAIN ANALYZE */ if (es->wal && !es->analyze) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("EXPLAIN option %s requires ANALYZE", "WAL"))); - /* if the timing was not set explicitly, set default value */ - es->timing = (timing_set) ? es->timing : es->analyze; - - /* if the buffers was not set explicitly, set default value */ - es->buffers = (buffers_set) ? es->buffers : es->analyze; - /* check that timing is used with EXPLAIN ANALYZE */ if (es->timing && !es->analyze) ereport(ERROR, @@ -317,9 +328,6 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("EXPLAIN options ANALYZE and GENERIC_PLAN cannot be used together"))); - /* if the summary was not set explicitly, set default value */ - es->summary = (summary_set) ? es->summary : es->analyze; - query = castNode(Query, stmt->query); if (IsQueryIdEnabled()) jstate = JumbleQuery(query); diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index aa5872bc15..d1faac97e8 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -144,4 +144,6 @@ extern void ExplainCloseGroup(const char *objtype, const char *labelname, extern DestReceiver *CreateExplainSerializeDestReceiver(ExplainState *es); +extern ExplainState *ParseExplainStmtOptions(ParseState *pstate, List *options); + #endif /* EXPLAIN_H */ diff --git a/src/include/pg_config_ext.h b/src/include/pg_config_ext.h new file mode 100644 index 0000000000..b4c07dd857 --- /dev/null +++ b/src/include/pg_config_ext.h @@ -0,0 +1,8 @@ +/* src/include/pg_config_ext.h. Generated from pg_config_ext.h.in by configure. */ +/* + * src/include/pg_config_ext.h.in. This is generated manually, not by + * autoheader, since we want to limit which symbols get defined here. + */ + +/* Define to the name of a signed 64-bit integer type. */ +#define PG_INT64_TYPE long int diff --git a/src/include/stamp-ext-h b/src/include/stamp-ext-h new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/src/include/stamp-ext-h @@ -0,0 +1 @@ + -- 2.40.1