From c40fce95dfb66faf4aae319f1a424dd4eb39028a Mon Sep 17 00:00:00 2001 From: Beena Emerson Date: Wed, 29 Nov 2017 16:58:47 +0530 Subject: [PATCH] Implement Runtime Partition Pruning Patch by: Beena Emerson, Dilip Kumar Discussion: https://postgr.es/m/CAOG9ApE16ac-_VVZVvv0gePSgkg_BwYEV1NBqZFqDR2bBE0X0A@mail.gmail.com --- src/backend/catalog/partition.c | 111 ++++++++++++-- src/backend/executor/nodeAppend.c | 263 ++++++++++++++++++++++++++++++-- src/backend/nodes/copyfuncs.c | 5 + src/backend/optimizer/path/allpaths.c | 150 +++++++++++++++--- src/backend/optimizer/path/joinrels.c | 2 +- src/backend/optimizer/plan/createplan.c | 24 +++ src/backend/optimizer/plan/planner.c | 2 +- src/backend/optimizer/prep/prepunion.c | 4 +- src/backend/optimizer/util/clauses.c | 14 ++ src/backend/optimizer/util/pathnode.c | 30 +++- src/backend/optimizer/util/relnode.c | 23 ++- src/backend/utils/cache/plancache.c | 2 +- src/include/catalog/partition.h | 6 +- src/include/nodes/execnodes.h | 6 + src/include/nodes/plannodes.h | 6 + src/include/nodes/relation.h | 12 ++ src/include/optimizer/clauses.h | 1 + src/include/optimizer/pathnode.h | 4 +- 18 files changed, 603 insertions(+), 62 deletions(-) diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c index c58c735..5a07a32 100644 --- a/src/backend/catalog/partition.c +++ b/src/backend/catalog/partition.c @@ -33,6 +33,7 @@ #include "catalog/pg_type.h" #include "commands/tablecmds.h" #include "executor/executor.h" +#include "executor/nodeSubplan.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -241,19 +242,22 @@ static void get_partition_dispatch_recurse(Relation rel, Relation parent, List **pds, List **leaf_part_oids); static Bitmapset *get_partitions_from_clauses_guts(Relation relation, - List *clauses); + List *clauses, ParamListInfo prmList, + ExprContext *econtext); static int classify_partition_bounding_keys(Relation relation, List *clauses, PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses); + List **or_clauses, + ParamListInfo prmList, ExprContext *econtext); static void remove_redundant_clauses(PartitionKey partkey, int partattoff, List *all_clauses, List **result, bool *constfalse); static bool partition_cmp_args(Oid partopfamily, Oid partopcintype, PartClause *op, PartClause *leftarg, PartClause *rightarg, bool *result); -static bool partkey_datum_from_expr(const Expr *expr, Datum *value); static Bitmapset *get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys); +static bool partkey_datum_from_expr(const Expr *expr, Datum *value, + ParamListInfo prmList, ExprContext *econtext); /* * RelationBuildPartitionDesc @@ -1536,6 +1540,34 @@ get_partition_dispatch_recurse(Relation rel, Relation parent, } } +void +get_leaf_part_recurse(Relation rel, List **leaf_part_oids) +{ + PartitionDesc partdesc = RelationGetPartitionDesc(rel); + int i; + + check_stack_depth(); + + for (i = 0; i < partdesc->nparts; i++) + { + Oid partrelid = partdesc->oids[i]; + + if (get_rel_relkind(partrelid) != RELKIND_PARTITIONED_TABLE) + *leaf_part_oids = lappend_oid(*leaf_part_oids, partrelid); + else + { + /* + * We assume all tables in the partition tree were already locked + * by the caller. + */ + Relation partrel = heap_open(partrelid, NoLock); + + get_leaf_part_recurse(partrel, leaf_part_oids); + heap_close(partrel, NoLock); + } + } +} + /* * get_partitions_from_clauses * Determine the set of partitions of relation that will satisfy all @@ -1545,7 +1577,8 @@ get_partition_dispatch_recurse(Relation rel, Relation parent, * A Bitmapset containing indexes of all selected partitions. */ Bitmapset * -get_partitions_from_clauses(Relation relation, List *partclauses) +get_partitions_from_clauses(Relation relation, List *partclauses, + ParamListInfo prmList, ExprContext *econtext) { Bitmapset *result; List *partconstr = RelationGetPartitionQual(relation); @@ -1554,7 +1587,7 @@ get_partitions_from_clauses(Relation relation, List *partclauses) partconstr = (List *) expression_planner((Expr *) partconstr); partclauses = list_concat(partclauses, partconstr); - result = get_partitions_from_clauses_guts(relation, partclauses); + result = get_partitions_from_clauses_guts(relation, partclauses, prmList, econtext); return result; } @@ -1569,7 +1602,8 @@ get_partitions_from_clauses(Relation relation, List *partclauses) * Return value is a Bitmapset containing the indexes of selected partitions. */ static Bitmapset * -get_partitions_from_clauses_guts(Relation relation, List *clauses) +get_partitions_from_clauses_guts(Relation relation, List *clauses, + ParamListInfo prmList, ExprContext *econtext) { PartitionDesc partdesc = RelationGetPartitionDesc(relation); Bitmapset *result = NULL; @@ -1581,7 +1615,8 @@ get_partitions_from_clauses_guts(Relation relation, List *clauses) nkeys = classify_partition_bounding_keys(relation, clauses, &keys, &constfalse, - &or_clauses); + &or_clauses, prmList, + econtext); /* * Only look up in the partition decriptor if the query provides * constraints on the keys at all. @@ -1604,7 +1639,8 @@ get_partitions_from_clauses_guts(Relation relation, List *clauses) Bitmapset *arg_partset; arg_partset = get_partitions_from_clauses_guts(relation, - list_make1(orarg)); + list_make1(orarg), + prmList, econtext); /* Combine partition sets obtained from mutually ORed clauses. */ or_partset = bms_union(or_partset, arg_partset); @@ -1645,7 +1681,8 @@ get_partitions_from_clauses_guts(Relation relation, List *clauses) static int classify_partition_bounding_keys(Relation relation, List *clauses, PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses) + List **or_clauses, + ParamListInfo prmList, ExprContext *econtext) { PartitionKey partkey = RelationGetPartitionKey(relation); int i; @@ -1694,6 +1731,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, continue; } } + else if (IsA(lfirst(lc), ExprState)) + clause = ((ExprState *) lfirst(lc))->expr; else clause = (Expr *) lfirst(lc); @@ -2138,7 +2177,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, n_datums_resolved = 0; for (i = 0; i < n_eqkeys; i++) { - if (partkey_datum_from_expr(eqkey_exprs[i], &value)) + if (partkey_datum_from_expr(eqkey_exprs[i], &value, + prmList, econtext)) { keys->eqkeys[i] = value; n_datums_resolved++; @@ -2150,7 +2190,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, n_datums_resolved = 0; for (i = 0; i < n_minkeys; i++) { - if (partkey_datum_from_expr(minkey_exprs[i], &value)) + if (partkey_datum_from_expr(minkey_exprs[i], &value, + prmList, econtext)) { keys->minkeys[i] = value; n_datums_resolved++; @@ -2163,7 +2204,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, n_datums_resolved = 0; for (i = 0; i < n_maxkeys; i++) { - if (partkey_datum_from_expr(maxkey_exprs[i], &value)) + if (partkey_datum_from_expr(maxkey_exprs[i], &value, + prmList, econtext)) { keys->maxkeys[i] = value; n_datums_resolved++; @@ -2186,7 +2228,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, * Extract constant value from expr and set *datum to that value */ static bool -partkey_datum_from_expr(const Expr *expr, Datum *value) +partkey_datum_from_expr(const Expr *expr, Datum *value, ParamListInfo prmList, + ExprContext *econtext) { /* * Add more expression types here as needed to support higher-level @@ -2198,6 +2241,42 @@ partkey_datum_from_expr(const Expr *expr, Datum *value) *value = ((Const *) expr)->constvalue; return true; + case T_Param: + switch (((Param *) expr)->paramkind) + { + case PARAM_EXTERN: + if (prmList) + { + Node *n = eval_const_expressions_from_list(prmList, (Node *) expr); + + if (IsA(n, Const)) + { + *value = ((Const *) n)->constvalue; + return true; + } + } + return false; + case PARAM_EXEC: + if (econtext) + { + Param *param = (Param *) expr; + ParamExecData *prm; + + prm = &(econtext->ecxt_param_exec_vals[param->paramid]); + if (unlikely(prm->execPlan != NULL)) + { + ExecSetParamPlan(prm->execPlan, econtext); + Assert(prm->execPlan == NULL); + } + *value = prm->value; + return true; + } + + default: + return false; + } + return false; + default: return false; } @@ -2353,9 +2432,11 @@ partition_cmp_args(Oid partopfamily, Oid partopcintype, rightarg_const; Assert(op->valid_cache && leftarg->valid_cache && rightarg->valid_cache); - if (!partkey_datum_from_expr(leftarg->constarg, &leftarg_const)) + if (!partkey_datum_from_expr(leftarg->constarg, &leftarg_const, + NULL, NULL)) return false; - if (!partkey_datum_from_expr(rightarg->constarg, &rightarg_const)) + if (!partkey_datum_from_expr(rightarg->constarg, &rightarg_const, + NULL, NULL)) return false; /* diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 1d2fb35..2890b3a 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -57,9 +57,12 @@ #include "postgres.h" +#include "catalog/pg_inherits_fn.h" +#include "nodes/relation.h" #include "executor/execdebug.h" #include "executor/nodeAppend.h" #include "miscadmin.h" +#include "optimizer/clauses.h" static TupleTableSlot *ExecAppend(PlanState *pstate); static bool exec_append_initialize_next(AppendState *appendstate); @@ -107,6 +110,24 @@ exec_append_initialize_next(AppendState *appendstate) } } +static List * +initClauses(PlanState *parent, List *old_list) +{ + List *new_list = NIL; + ListCell *lc; + + if (old_list == NULL) + return NULL; + + foreach(lc, old_list) + { + Expr *val = (Expr *) lfirst(lc); + + new_list = lappend(new_list, ExecInitExpr(val, parent)); + } + return new_list; +} + /* ---------------------------------------------------------------- * ExecInitAppend * @@ -165,25 +186,134 @@ ExecInitAppend(Append *node, EState *estate, int eflags) */ ExecInitResultTupleSlot(estate, &appendstate->ps); - /* - * call ExecInitNode on each of the plans to be executed and save the - * results into the array "appendplans". - */ - i = 0; - foreach(lc, node->appendplans) + if (node->extern_quals) { - Plan *initNode = (Plan *) lfirst(lc); + List *temp_appendplans = NIL; + Append *temp_node; + Relation rel; + Bitmapset *partset; + Bitmapset *subplan_indexes; + int i, + cur_index; + PartitionDispatch *pd, + *p1, + parent; + int num_parted; + List *leaf_part_oids = NIL; + List *parents = NIL; + + rel = relation_open(node->parentoid, NoLock); + + /* + * Get the information about the partition tree after locking all the + * partitions. + */ + (void) find_all_inheritors(RelationGetRelid(rel), AccessShareLock, NULL); + pd = RelationGetPartitionDispatchInfo(rel, &num_parted, &leaf_part_oids); + parents = lappend(parents, &pd[0]); + do + { + p1 = linitial(parents); + parent = *p1; + + partset = get_partitions_from_clauses(parent->reldesc, + list_copy(node->extern_quals), + estate->es_param_list_info, + NULL); + + if (!bms_is_empty(partset)) + { + i = 0; + while ((cur_index = bms_first_member(partset)) >= 0) + { + if (cur_index < 0) + { + // PartitionDesc *partdesc = parent->partdesc; + + // subplan_indexes = bms_make_singleton(partdesc->boundinfo->default_index); + } + + else if (parent->indexes[cur_index] >= 0) + { + subplan_indexes = bms_make_singleton(parent->indexes[cur_index]); + break; + } + else + parents = lappend(parents, &pd[-parent->indexes[cur_index]]); + } + } + + parents = list_delete_first(parents); + } while (parents); + + for (i = 1; i < num_parted; i++) + { + PartitionDispatch partdispatch = pd[i]; + + heap_close(partdispatch->reldesc, NoLock); + ExecDropSingleTupleTableSlot(partdispatch->tupslot); + } + + if (!bms_is_empty(subplan_indexes)) + { + int j; + int i = 0; + + while ((j = bms_first_member(subplan_indexes)) >= 0) + { + int index = node->append_paths_array[j]; + + if (index >= 0) + { + Plan *initNode = (Plan *) list_nth(node->appendplans, + index); + + temp_appendplans = lappend(temp_appendplans, initNode); + appendplanstates[i++] = ExecInitNode(initNode, estate, + eflags); + } + } + } - appendplanstates[i] = ExecInitNode(initNode, estate, eflags); - i++; + relation_close(rel, NoLock); + /* create new AppendState for our append node */ + temp_node = copyObject(node); + temp_node->appendplans = temp_appendplans; + ((Plan *) temp_node)->plan_rows = i; + appendstate->as_nplans = i; + appendstate->ps.plan = (Plan *) temp_node; + appendstate->ps.state = estate; + appendstate->ps.ExecProcNode = ExecAppend; + appendstate->appendplans = appendplanstates; } + else + { + appendstate->parentoid = node->parentoid; + ExecAssignExprContext(estate, &appendstate->ps); - /* - * initialize output tuple type - */ + /* + * call ExecInitNode on each of the plans to be executed and save the + * results into the array "appendplans". + */ + i = 0; + foreach(lc, node->appendplans) + { + Plan *initNode = (Plan *) lfirst(lc); + + appendplanstates[i] = ExecInitNode(initNode, estate, eflags); + i++; + } + } + + if (node->join_clauses) + appendstate->join_clauses = initClauses((PlanState *) appendstate, + node->join_clauses); ExecAssignResultTypeFromTL(&appendstate->ps); appendstate->ps.ps_ProjInfo = NULL; + appendstate->array_size = node->array_size; + appendstate->append_paths_array = node->append_paths_array; + /* * initialize to scan first subplan */ @@ -237,7 +367,22 @@ ExecAppend(PlanState *pstate) * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + /* + * For runtime partition pruning, goto the next valid partition + * index + */ + if (node->index) + { + if (++node->as_whichpartition < list_length(node->index)) + node->as_whichplan = list_nth_int(node->index, + node->as_whichpartition); + + return ExecClearTuple(node->ps.ps_ResultTupleSlot); + } + else + node->as_whichplan++; + } else node->as_whichplan--; if (!exec_append_initialize_next(node)) @@ -280,6 +425,89 @@ ExecReScanAppend(AppendState *node) { int i; + if (node->ps.chgParam != NULL && node->join_clauses) + { + Bitmapset *partset, + *subplan_indexes = NULL; + Relation rel; + PartitionDispatch *pd, + parent, + *p1; + List *parents = NIL; + List *leaf_part_oids = NIL; + int cur_index, + num_parted; + + rel = relation_open(node->parentoid, NoLock); + + /* + * Get the information about the partition tree after locking all the + * partitions. + */ + (void) find_all_inheritors(RelationGetRelid(rel), AccessShareLock, NULL); + pd = RelationGetPartitionDispatchInfo(rel, &num_parted, &leaf_part_oids); + relation_close(rel, NoLock); + parents = lappend(parents, &pd[0]); + node->index = NIL; + do + { + p1 = linitial(parents); + parent = *p1; + + partset = get_partitions_from_clauses(parent->reldesc, + list_copy(node->join_clauses), + NULL, + node->ps.ps_ExprContext); + + if (!bms_is_empty(partset)) + { + i = 0; + while ((cur_index = bms_first_member(partset)) >= 0) + { + if (cur_index < 0) + { + //PartitionDesc *partdesc = parent->partdesc; + + // subplan_indexes = bms_make_singleton(partdesc->boundinfo->default_index); + } + + else if (parent->indexes[cur_index] >= 0) + { + subplan_indexes = + bms_make_singleton(parent->indexes[cur_index]); + break; + } + else + parents = + lappend(parents, &pd[-parent->indexes[cur_index]]); + } + } + + parents = list_delete_first(parents); + } while (parents); + + for (i = 1; i < num_parted; i++) + { + PartitionDispatch partdispatch = pd[i]; + + heap_close(partdispatch->reldesc, NoLock); + ExecDropSingleTupleTableSlot(partdispatch->tupslot); + } + + if (!bms_is_empty(subplan_indexes)) + { + int j; + + while ((j = bms_first_member(subplan_indexes)) >= 0) + { + int index = node->append_paths_array[j]; + + if (index >= 0) + node->index = lappend_int(node->index, index); + } + } + } + for (i = 0; i < node->as_nplans; i++) { PlanState *subnode = node->appendplans[i]; @@ -298,6 +526,13 @@ ExecReScanAppend(AppendState *node) if (subnode->chgParam == NULL) ExecReScan(subnode); } - node->as_whichplan = 0; + + if (node->index) + { + node->as_whichplan = linitial_int(node->index); + node->as_whichpartition = 0; + } + else + node->as_whichplan = 0; exec_append_initialize_next(node); } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index c1a83ca..63b206f 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -242,6 +242,11 @@ _copyAppend(const Append *from) */ COPY_NODE_FIELD(partitioned_rels); COPY_NODE_FIELD(appendplans); + COPY_NODE_FIELD(extern_quals); + COPY_NODE_FIELD(join_clauses); + COPY_SCALAR_FIELD(parentoid); + COPY_POINTER_FIELD(append_paths_array, from->array_size * sizeof(int)); + COPY_SCALAR_FIELD(array_size); return newnode; } diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index a3d0468..b7db443 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -18,10 +18,12 @@ #include #include +#include "catalog/partition.h" #include "access/sysattr.h" #include "access/tsmapi.h" #include "catalog/partition.h" #include "catalog/pg_class.h" +#include "catalog/pg_inherits_fn.h" #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" @@ -140,11 +142,13 @@ static void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); static List *get_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte); + RangeTblEntry *rte, List *chk_clauses); static List *match_clauses_to_partkey(RelOptInfo *rel, List *clauses, + bool *contains_param, bool *contains_const, bool *constfalse); +static int list_member_oid_index(List *list, Oid datum); /* @@ -288,6 +292,22 @@ set_base_rel_sizes(PlannerInfo *root) if (root->glob->parallelModeOK) set_rel_consider_parallel(root, rel, rte); + if (rte->relkind == RELKIND_PARTITIONED_TABLE) + { + List *leaf_parts = NIL; + Relation parent = relation_open(rte->relid, NoLock); + int i; + + (void) find_all_inheritors(RelationGetRelid(parent), AccessShareLock, NULL); + get_leaf_part_recurse(parent, &leaf_parts); + root->array_size = list_length(leaf_parts); + root->leaf_node_oids = leaf_parts; + root->append_paths_array = palloc0(root->array_size * sizeof(int)); + for (i = 0; i < root->array_size; i++) + root->append_paths_array[i] = -1; + root->append_count = 0; + relation_close(parent, NoLock); + } set_rel_size(root, rel, rti, rte); } } @@ -349,6 +369,41 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, { /* It's an "append relation", process accordingly */ set_append_rel_size(root, rel, rti, rte); + + if (rel->joininfo && rel->part_scheme) + { + List *partclauses; + bool contains_param, + contains_const, + constfalse; + + /* + * Get the clauses that match the partition key, including + * information about any nullness tests against partition keys. + * Set keynullness to a invalid value of NullTestType, which 0 is + * not. + */ + partclauses = match_clauses_to_partkey(rel, + list_copy(rel->joininfo), + &contains_param, + &contains_const, + &constfalse); + + if (partclauses != NIL) + { + ListCell *lc; + + foreach(lc, partclauses) + { + Node *n = lfirst(lc); + + if (!list_member(root->join_append_clauses, n)); + root->join_append_clauses = lappend(root->join_append_clauses, n); + } + } + + } + } else { @@ -855,6 +910,7 @@ set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) rel->fdwroutine->GetForeignPaths(root, rel, rte->relid); } + /* * get_append_rel_partitions * Return the list of partitions of rel that pass the clauses mentioned @@ -865,24 +921,26 @@ set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) static List * get_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte) + RangeTblEntry *rte, List *chk_clauses) { Relation parent = heap_open(rte->relid, NoLock); PartitionDesc partdesc = RelationGetPartitionDesc(parent); - List *partclauses; - List *result = NIL; - int i; - Bitmapset *partindexes = NULL; - bool contains_const, - constfalse; + List *partclauses; + List *result = NIL; + int i; + Bitmapset *partindexes = NULL; + bool contains_param, + contains_const, + constfalse; /* * Get the clauses that match the partition key, including information - * about any nullness tests against partition keys. Set keynullness to - * a invalid value of NullTestType, which 0 is not. + * about any nullness tests against partition keys. Set keynullness to a + * invalid value of NullTestType, which 0 is not. */ partclauses = match_clauses_to_partkey(rel, list_copy(rel->baserestrictinfo), + &contains_param, &contains_const, &constfalse); @@ -891,11 +949,26 @@ get_append_rel_partitions(PlannerInfo *root, * the same to prune partitions right away. */ if (partclauses != NIL && contains_const && !constfalse) - partindexes = get_partitions_from_clauses(parent, partclauses); - else if (!constfalse) + partindexes = get_partitions_from_clauses(parent, partclauses, NULL, NULL); + else if (!constfalse && !contains_param) /* No clauses to prune paritions, so scan all partitions. */ partindexes = bms_add_range(partindexes, 0, partdesc->nparts - 1); + if (partclauses != NIL && contains_param) + { + ListCell *lc; + + foreach(lc, partclauses) + { + Node *n = lfirst(lc); + + if (!list_member(root->extern_clauses, n)); + root->extern_clauses = lappend(root->extern_clauses, n); + } + if (!partindexes) + partindexes = bms_add_range(partindexes, 0, partdesc->nparts - 1); + } + /* Fetch the partition appinfos. */ while ((i = bms_first_member(partindexes)) >= 0) { @@ -951,10 +1024,13 @@ get_append_rel_partitions(PlannerInfo *root, * * If the list contains a pseudo-constant RestrictInfo with constant false * value, *constfalse is set. + * + * If the list has a param, *contains_param is set */ static List * match_clauses_to_partkey(RelOptInfo *rel, List *clauses, + bool *contains_param, bool *contains_const, bool *constfalse) { @@ -962,6 +1038,7 @@ match_clauses_to_partkey(RelOptInfo *rel, List *result = NIL; ListCell *lc; + *contains_param = false; *contains_const = false; *constfalse = false; @@ -1002,6 +1079,7 @@ match_clauses_to_partkey(RelOptInfo *rel, if (or_clause((Node *) clause) && match_clauses_to_partkey(rel, list_copy(((BoolExpr *) clause)->args), + contains_param, &contains_const1, &constfalse1) != NIL) { @@ -1079,6 +1157,9 @@ match_clauses_to_partkey(RelOptInfo *rel, /* Neither argument matches the partition key. */ continue; + if (IsA(constexpr, Param)) + *contains_param = true; + /* * Only allow strict operators to think sanely about the * behavior with null arguments. @@ -1230,6 +1311,22 @@ match_clauses_to_partkey(RelOptInfo *rel, return result; } +static int +list_member_oid_index(List *list, Oid datum) +{ + int i = 0; + const ListCell *cell; + + foreach(cell, list) + { + if (lfirst_oid(cell) == datum) + return i; + i++; + } + + return -1; +} + /* * set_append_rel_size * Set size estimates for a simple "append relation" @@ -1245,7 +1342,8 @@ static void set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { - List *rel_appinfos = NIL; + List *rel_appinfos = NIL, + *chk_clauses = NIL; int parentRTindex = rti; bool has_live_children; double parent_rows; @@ -1261,9 +1359,9 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, if (rte->relkind != RELKIND_PARTITIONED_TABLE) { - foreach (l, root->append_rel_list) + foreach(l, root->append_rel_list) { - AppendRelInfo *appinfo = lfirst(l); + AppendRelInfo *appinfo = lfirst(l); /* append_rel_list contains all append rels; ignore others */ if (appinfo->parent_relid == parentRTindex) @@ -1272,7 +1370,7 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, } else { - rel_appinfos = get_append_rel_partitions(root, rel, rte); + rel_appinfos = get_append_rel_partitions(root, rel, rte, chk_clauses); rel->live_partitioned_rels = list_make1_int(rti); } @@ -1495,8 +1593,8 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, if (childrel->part_scheme && rel->part_scheme) { rel->live_partitioned_rels = - list_concat(rel->live_partitioned_rels, - list_copy(childrel->live_partitioned_rels)); + list_concat(rel->live_partitioned_rels, + list_copy(childrel->live_partitioned_rels)); } /* @@ -1623,6 +1721,7 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, int childRTindex; RangeTblEntry *childRTE; RelOptInfo *childrel; + int index; /* append_rel_list contains all append rels; ignore others */ if (appinfo->parent_relid != parentRTindex) @@ -1653,6 +1752,11 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, if (IS_DUMMY_REL(childrel)) continue; + /* Only consider non dummy children */ + index = list_member_oid_index(root->leaf_node_oids, childRTE->relid); + if (index >= 0) + root->append_paths_array[index] = root->append_count++; + /* * Child is live, so add it to the live_childrels list for use below. */ @@ -1714,7 +1818,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, rte = planner_rt_fetch(rel->relid, root); if (rte->rtekind == RTE_RELATION && rte->relkind == RELKIND_PARTITIONED_TABLE) - partitioned_rels = rel->live_partitioned_rels; + partitioned_rels = rel->live_partitioned_rels; } else if (rel->reloptkind == RELOPT_JOINREL && rel->part_scheme) { @@ -1834,7 +1938,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, * if we have zero or one live subpath due to constraint exclusion.) */ if (subpaths_valid) - add_path(rel, (Path *) create_append_path(rel, subpaths, NULL, 0, + add_path(rel, (Path *) create_append_path(root, rel, subpaths, NULL, 0, partitioned_rels)); /* @@ -1861,7 +1965,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, Assert(parallel_workers > 0); /* Generate a partial append path. */ - appendpath = create_append_path(rel, partial_subpaths, NULL, + appendpath = create_append_path(root, rel, partial_subpaths, NULL, parallel_workers, partitioned_rels); add_partial_path(rel, (Path *) appendpath); } @@ -1915,7 +2019,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (subpaths_valid) add_path(rel, (Path *) - create_append_path(rel, subpaths, required_outer, 0, + create_append_path(root, rel, subpaths, required_outer, 0, partitioned_rels)); } } @@ -2152,7 +2256,7 @@ set_dummy_rel_pathlist(RelOptInfo *rel) rel->pathlist = NIL; rel->partial_pathlist = NIL; - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NULL, 0, NIL)); /* * We set the cheapest path immediately, to ensure that IS_DUMMY_REL() diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 7356683..af7e790 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -1232,7 +1232,7 @@ mark_dummy_rel(RelOptInfo *rel) rel->partial_pathlist = NIL; /* Set up the dummy path */ - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NULL, 0, NIL)); /* Set or update cheapest_total_path and related fields */ set_cheapest(rel); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 9c74e39..1670998 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1001,6 +1001,24 @@ create_join_plan(PlannerInfo *root, JoinPath *best_path) return plan; } +static List * +replace_partition_nestloop_params(PlannerInfo *root, List *old_list) +{ + List *new_list = NIL; + ListCell *lc; + + if (old_list == NULL) + return NULL; + + foreach(lc, old_list) + { + Node *n = lfirst(lc); + + new_list = lappend(new_list, replace_nestloop_params(root, n)); + } + return new_list; +} + /* * create_append_plan * Create an Append plan for 'best_path' and (recursively) plans @@ -1063,6 +1081,12 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path) copy_generic_path_info(&plan->plan, (Path *) best_path); + plan->extern_quals = best_path->extern_quals; + plan->join_clauses = replace_partition_nestloop_params(root, best_path->join_clauses); + plan->parentoid = best_path->parentoid; + plan->array_size = best_path->array_size; + plan->append_paths_array = best_path->append_paths_array; + return (Plan *) plan; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index fd0e483..8a8876f 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3675,7 +3675,7 @@ create_grouping_paths(PlannerInfo *root, paths = lappend(paths, path); } path = (Path *) - create_append_path(grouped_rel, + create_append_path(root, grouped_rel, paths, NULL, 0, diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index f620243..12d0f85 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -590,7 +590,7 @@ generate_union_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NULL, 0, NIL); + path = (Path *) create_append_path(root, result_rel, pathlist, NULL, 0, NIL); /* We have to manually jam the right tlist into the path; ick */ path->pathtarget = create_pathtarget(root, tlist); @@ -702,7 +702,7 @@ generate_nonunion_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NULL, 0, NIL); + path = (Path *) create_append_path(root, result_rel, pathlist, NULL, 0, NIL); /* We have to manually jam the right tlist into the path; ick */ path->pathtarget = create_pathtarget(root, tlist); diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 30cdd3d..2f46932 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -5197,3 +5197,17 @@ tlist_matches_coltypelist(List *tlist, List *coltypelist) return true; } + +Node * +eval_const_expressions_from_list(ParamListInfo prmList, Node *node) +{ + eval_const_expressions_context context; + + if (prmList) + context.boundParams = prmList; /* bound Params */ + context.root = NULL; + context.active_fns = NIL; /* nothing being recursively simplified */ + context.case_val = NULL; /* no CASE being examined */ + context.estimate = false; /* safe transformations only */ + return eval_const_expressions_mutator(node, &context); +} diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 36ec025..889419d 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1208,7 +1208,7 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, * Note that we must handle subpaths = NIL, representing a dummy access path. */ AppendPath * -create_append_path(RelOptInfo *rel, List *subpaths, Relids required_outer, +create_append_path(PlannerInfo *root, RelOptInfo *rel, List *subpaths, Relids required_outer, int parallel_workers, List *partitioned_rels) { AppendPath *pathnode = makeNode(AppendPath); @@ -1253,6 +1253,34 @@ create_append_path(RelOptInfo *rel, List *subpaths, Relids required_outer, Assert(bms_equal(PATH_REQ_OUTER(subpath), required_outer)); } + if (root) + { + pathnode->append_paths_array = root->append_paths_array; + pathnode->array_size = root->array_size; + + } + if (root && (required_outer || root->extern_clauses)) + { + RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); + + if (rte && rte->rtekind == RTE_RELATION) + { + Oid poid = rte->relid; + Relation prel = relation_open(poid, NoLock); + + if (prel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + if (root->join_append_clauses) + pathnode->join_clauses = root->join_append_clauses; + /* Set et_keys for extern params */ + if (root->extern_clauses) + pathnode->extern_quals = root->extern_clauses; + pathnode->parentoid = poid; + } + relation_close(prel, NoLock); + } + } + return pathnode; } diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index b06696b..9abc79c 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -1556,6 +1556,9 @@ ParamPathInfo * get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) { ParamPathInfo *ppi; + Relids joinrelids; + List *pclauses; + ListCell *lc; /* Unparameterized paths have no ParamPathInfo */ if (bms_is_empty(required_outer)) @@ -1567,11 +1570,29 @@ get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) if ((ppi = find_param_path_info(appendrel, required_outer))) return ppi; + + /* + * Generally for appendrel we don't fetch the clause from the join clause + * (only we do so for baserel), but for identifying whether the appendrel + * is applicable for runtime pruning or not. + */ + joinrelids = bms_union(appendrel->relids, required_outer); + pclauses = NIL; + foreach(lc, appendrel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + if (join_clause_is_movable_into(rinfo, + appendrel->relids, + joinrelids)) + pclauses = lappend(pclauses, rinfo); + } + /* Else build the ParamPathInfo */ ppi = makeNode(ParamPathInfo); ppi->ppi_req_outer = required_outer; ppi->ppi_rows = 0; - ppi->ppi_clauses = NIL; + ppi->ppi_clauses = pclauses; appendrel->ppilist = lappend(appendrel->ppilist, ppi); return ppi; diff --git a/src/include/catalog/partition.h b/src/include/catalog/partition.h index 81c626f..2bad8f1 100644 --- a/src/include/catalog/partition.h +++ b/src/include/catalog/partition.h @@ -16,6 +16,7 @@ #include "fmgr.h" #include "executor/tuptable.h" #include "nodes/execnodes.h" +#include "nodes/relation.h" #include "parser/parse_node.h" #include "utils/rel.h" @@ -110,5 +111,8 @@ extern List *get_proposed_default_constraint(List *new_part_constaints); /* For partition-pruning */ extern Bitmapset *get_partitions_from_clauses(Relation relation, - List *partclauses); + List *partclauses, + ParamListInfo prmList, + ExprContext *econtext); +extern void get_leaf_part_recurse(Relation rel, List **leaf_part_oids); #endif /* PARTITION_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e05bc04..620fc25 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1006,6 +1006,12 @@ typedef struct AppendState PlanState **appendplans; /* array of PlanStates for my inputs */ int as_nplans; int as_whichplan; + Oid parentoid; + List *index; /* subplan indexes to scan for runtime pruning */ + int as_whichpartition; /* current partition scanned from list */ + List *join_clauses; + int *append_paths_array; + int array_size; } AppendState; /* ---------------- diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index dd74efa..217a841 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -19,6 +19,7 @@ #include "nodes/bitmapset.h" #include "nodes/lockoptions.h" #include "nodes/primnodes.h" +#include "nodes/relation.h" /* ---------------------------------------------------------------- @@ -248,6 +249,11 @@ typedef struct Append /* RT indexes of non-leaf tables in a partition tree */ List *partitioned_rels; List *appendplans; + Oid parentoid; + List *extern_quals; + List *join_clauses; + int *append_paths_array; + int array_size; } Append; /* ---------------- diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 9c67bd1..166e964 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -168,6 +168,7 @@ typedef struct PlannerInfo */ List *plan_params; /* list of PlannerParamItems, see below */ Bitmapset *outer_params; + List *extern_clauses; /* * simple_rel_array holds pointers to "base rels" and "other rels" (see @@ -317,6 +318,12 @@ typedef struct PlannerInfo /* optional private data for join_search_hook, e.g., GEQO */ void *join_search_private; + + List *leaf_node_oids; + List *join_append_clauses; + int array_size; + int append_count; + int *append_paths_array; } PlannerInfo; @@ -1289,6 +1296,11 @@ typedef struct AppendPath /* RT indexes of non-leaf tables in a partition tree */ List *partitioned_rels; List *subpaths; /* list of component Paths */ + List *extern_quals; + List *join_clauses; + int *append_paths_array; + int array_size; + Oid parentoid; } AppendPath; #define IS_DUMMY_PATH(p) \ diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index e367221..c7f5262 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -79,6 +79,7 @@ extern void CommuteOpExpr(OpExpr *clause); extern void CommuteRowCompareExpr(RowCompareExpr *clause); extern Node *eval_const_expressions(PlannerInfo *root, Node *node); +extern Node *eval_const_expressions_from_list(ParamListInfo prm_list, Node *node); extern Node *estimate_expression_value(PlannerInfo *root, Node *node); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index c1f2fc9..7c1fbfd 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -63,8 +63,8 @@ extern BitmapOrPath *create_bitmap_or_path(PlannerInfo *root, List *bitmapquals); extern TidPath *create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, Relids required_outer); -extern AppendPath *create_append_path(RelOptInfo *rel, List *subpaths, - Relids required_outer, int parallel_workers, +extern AppendPath *create_append_path(PlannerInfo *root, RelOptInfo *rel, + List *subpaths, Relids required_outer, int parallel_workers, List *partitioned_rels); extern MergeAppendPath *create_merge_append_path(PlannerInfo *root, RelOptInfo *rel, -- 1.8.3.1