From 6ee8c7bfe6c879f274e37c60d0e617d26dc01976 Mon Sep 17 00:00:00 2001 From: Beena Emerson Date: Thu, 9 Nov 2017 16:19:28 +0530 Subject: [PATCH] Implement runtime partiton pruning Patch by: Beena Emerson, Dilip Kumar Discussion: https://postgr.es/m/CAOG9ApE16ac-_VVZVvv0gePSgkg_BwYEV1NBqZFqDR2bBE0X0A@mail.gmail.com --- src/backend/catalog/partition.c | 122 +++---------- src/backend/executor/nodeAppend.c | 315 ++++++++++++++++++++++++++++++-- src/backend/nodes/copyfuncs.c | 27 +++ src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/allpaths.c | 72 ++++++-- src/backend/optimizer/path/joinrels.c | 2 +- src/backend/optimizer/plan/createplan.c | 46 +++++ src/backend/optimizer/plan/planner.c | 2 +- src/backend/optimizer/prep/prepunion.c | 4 +- src/backend/optimizer/util/clauses.c | 16 ++ src/backend/optimizer/util/pathnode.c | 36 +++- src/backend/optimizer/util/relnode.c | 23 ++- src/backend/utils/cache/plancache.c | 2 +- src/include/catalog/partition.h | 8 + src/include/nodes/execnodes.h | 5 + src/include/nodes/nodes.h | 1 + src/include/nodes/plannodes.h | 4 + src/include/nodes/relation.h | 87 +++++++++ src/include/optimizer/clauses.h | 1 + src/include/optimizer/pathnode.h | 2 +- src/include/optimizer/paths.h | 4 + 21 files changed, 652 insertions(+), 129 deletions(-) diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c index 152ae60..073164d 100644 --- a/src/backend/catalog/partition.c +++ b/src/backend/catalog/partition.c @@ -130,84 +130,6 @@ typedef struct } PartClause; /* - * PartScanKeyInfo - * Bounding scan keys to look up a table's partitions obtained from - * mutually-ANDed clauses containing partitioning-compatible operators - */ -typedef struct PartScanKeyInfo -{ - /* - * Constants constituting the *whole* partition key compared using - * partitioning-compatible equality operator(s). When n_eqkeys > 0, other - * keys (minkeys and maxkeys) are irrelevant. - */ - Datum eqkeys[PARTITION_MAX_KEYS]; - int n_eqkeys; - - /* - * Constants that constitute the lower bound on the partition key or a - * prefix thereof. The last of those constants is compared using > or >= - * operator compatible with partitioning, making this the lower bound in - * a range query. - */ - Datum minkeys[PARTITION_MAX_KEYS]; - int n_minkeys; - bool min_incl; - - /* - * Constants that constitute the upper bound on the partition key or a - * prefix thereof. The last of those constants is compared using < or <= - * operator compatible with partitioning, making this the upper bound in - * a range query. - */ - Datum maxkeys[PARTITION_MAX_KEYS]; - int n_maxkeys; - bool max_incl; - - /* - * Specifies the type of NullTest that was applied to each of the - * partition key columns or -1 if none was applied. Partitioning handles - * null partition keys specially depending on the partitioning method in - * use, so get_partitions_for_keys can return partitions according to - * the nullness condition for partition keys. - */ - NullTestType keynullness[PARTITION_MAX_KEYS]; -} PartScanKeyInfo; - - /* A data structure to represent a partition set. */ -typedef struct PartitionSet -{ - /* - * If either empty or all_parts is true, values of the other fields are - * invalid. - */ - bool empty; /* contains no partitions */ - bool all_parts; /* contains all partitions */ - - /* - * In the case of range partitioning, min_part_index contains the index of - * the lowest partition contained in the set and max_datum_index that of - * the highest partition (all partitions between these two indexes - * inclusive are part of the set.) Since other types of partitioning do - * not impose order on the data contained in successive partitions, these - * fields are not set in that case. - */ - bool use_range; - int min_part_idx; - int max_part_idx; - - /* - * other_parts contains the indexes of partitions that are not covered by - * the range defined by min/max indexes. For example, in the case of - * range partitoning, it will include default partition index (if any). - * Also, this is the only way to return list partitions, because list - * partitions do not have the same ordering property as range partitions, - * so it's pointless to use the min/max range method. - */ - Bitmapset *other_parts; -} PartitionSet; - -/* * PartitionBoundCmpArg - Caller-defined argument to be passed to * partition_bound_cmp() * @@ -277,9 +199,6 @@ static PartitionSet *partset_copy(const PartitionSet *in); static PartitionSet *partset_intersect(PartitionSet *a, const PartitionSet *b); static PartitionSet *partset_union(PartitionSet *a, const PartitionSet *b); static PartitionSet *partset_new(bool empty, bool all_parts); -static int classify_partition_bounding_keys(Relation relation, List *clauses, - PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses); static void remove_redundant_clauses(PartitionKey partkey, int partattoff, List *all_clauses, List **result, bool *constfalse); @@ -287,8 +206,6 @@ static bool partition_cmp_args(Oid partopfamily, Oid partopcintype, PartClause *op, PartClause *leftarg, PartClause *rightarg, bool *result); static bool partkey_datum_from_expr(const Expr *expr, Datum *value); -static PartitionSet *get_partitions_for_keys(Relation rel, - PartScanKeyInfo *keys); /* * RelationBuildPartitionDesc @@ -1641,9 +1558,11 @@ get_partitions_from_clauses_guts(Relation relation, List *clauses) List *or_clauses; ListCell *lc; + memset(&keys, 0, sizeof(PartScanKeyInfo)); + nkeys = classify_partition_bounding_keys(relation, clauses, &keys, &constfalse, - &or_clauses); + &or_clauses, false); if (constfalse) /* None of the partitions will satisfy the clauses. */ partset = partset_new(true, false); @@ -1875,10 +1794,10 @@ partset_union(PartitionSet *a, const PartitionSet *b) * the responsibility of the caller to process the argument clauses of each of * the OR clauses, which would involve recursively calling this function. */ -static int +int classify_partition_bounding_keys(Relation relation, List *clauses, PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses) + List **or_clauses, bool is_runtime) { PartitionKey partkey = RelationGetPartitionKey(relation); int i; @@ -2314,7 +2233,6 @@ classify_partition_bounding_keys(Relation relation, List *clauses, n_eqkeys = 0; /* Populate keys. */ - memset(keys, 0, sizeof(PartScanKeyInfo)); if (n_eqkeys + n_minkeys + n_maxkeys + n_keynullness > 0) { Datum value; @@ -2325,7 +2243,10 @@ classify_partition_bounding_keys(Relation relation, List *clauses, { if (partkey_datum_from_expr(eqkey_exprs[i], &value)) { - keys->eqkeys[i] = value; + if (value) + keys->eqkeys_datums[i] = value; + else + keys->eqkeys = lappend(keys->eqkeys, eqkey_exprs[i]); n_datums_resolved++; } } @@ -2337,7 +2258,10 @@ classify_partition_bounding_keys(Relation relation, List *clauses, { if (partkey_datum_from_expr(minkey_exprs[i], &value)) { - keys->minkeys[i] = value; + if (value) + keys->minkeys_datums[i] = value; + else + keys->minkeys = lappend(keys->minkeys, minkey_exprs[i]); n_datums_resolved++; } } @@ -2350,7 +2274,10 @@ classify_partition_bounding_keys(Relation relation, List *clauses, { if (partkey_datum_from_expr(maxkey_exprs[i], &value)) { - keys->maxkeys[i] = value; + if (value) + keys->maxkeys_datums[i] = value; + else + keys->maxkeys = lappend(keys->maxkeys, maxkey_exprs[i]); n_datums_resolved++; } } @@ -2363,6 +2290,9 @@ classify_partition_bounding_keys(Relation relation, List *clauses, n_total += n_keynullness; } + if (n_total == 0) + keys = NULL; + return n_total; } @@ -2383,6 +2313,10 @@ partkey_datum_from_expr(const Expr *expr, Datum *value) *value = ((Const *) expr)->constvalue; return true; + case T_Param: + *value = '\0'; + return true; + default: return false; } @@ -2593,7 +2527,7 @@ partition_cmp_args(Oid partopfamily, Oid partopcintype, * Outputs: * Partition set satisfying the keys. */ -static PartitionSet * +PartitionSet * get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys) { PartitionSet *partset; @@ -2677,7 +2611,7 @@ get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys) if (keys->n_eqkeys > 0) { memset(&arg, 0, sizeof(PartitionBoundCmpArg)); - arg.datums = keys->eqkeys; + arg.datums = keys->eqkeys_datums; arg.ndatums = keys->n_eqkeys; eqoff = partition_bound_bsearch(partkey, boundinfo, &arg, &is_equal); @@ -2729,7 +2663,7 @@ get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys) if (keys->n_minkeys > 0) { memset(&arg, 0, sizeof(PartitionBoundCmpArg)); - arg.datums = keys->minkeys; + arg.datums = keys->minkeys_datums; arg.ndatums = keys->n_minkeys; minoff = partition_bound_bsearch(partkey, boundinfo, &arg, &is_equal); @@ -2815,7 +2749,7 @@ get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys) if (keys->n_maxkeys > 0) { memset(&arg, 0, sizeof(PartitionBoundCmpArg)); - arg.datums = keys->maxkeys; + arg.datums = keys->maxkeys_datums; arg.ndatums = keys->n_maxkeys; maxoff = partition_bound_bsearch(partkey, boundinfo, &arg, &is_equal); diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 1d2fb35..0aebc90 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -1,4 +1,4 @@ -/*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * * nodeAppend.c * routines to handle append nodes. @@ -57,9 +57,11 @@ #include "postgres.h" +#include "nodes/relation.h" #include "executor/execdebug.h" #include "executor/nodeAppend.h" #include "miscadmin.h" +#include "optimizer/clauses.h" static TupleTableSlot *ExecAppend(PlanState *pstate); static bool exec_append_initialize_next(AppendState *appendstate); @@ -108,6 +110,109 @@ exec_append_initialize_next(AppendState *appendstate) } /* ---------------------------------------------------------------- + * extern_eval_const_expressions + * + * Evalute the expressions to a constant. + * ---------------------------------------------------------------- + */ +static PartScanKeyInfo* +extern_eval_const_expressions(EState *estate, PartScanKeyInfo *old_keys) +{ + PartScanKeyInfo *keys; + int i; + ListCell *lc; + + if (old_keys == NULL) + return NULL; + + keys = makeNode(PartScanKeyInfo); + i = 0; + foreach(lc, old_keys->eqkeys) + { + Node *val = lfirst(lc); + Node *n = eval_const_expressions_from_list(estate->es_param_list_info, val); + + keys->eqkeys_datums[i++] = ((Const *) n)->constvalue; + } + keys->n_eqkeys = old_keys->n_eqkeys; + + i = 0; + foreach(lc, old_keys->minkeys) + { + Node *val = lfirst(lc); + Node *n = eval_const_expressions_from_list(estate->es_param_list_info, val); + + keys->minkeys_datums[i++] = ((Const *) n)->constvalue; + } + + keys->n_minkeys = old_keys->n_minkeys; + keys->min_incl = old_keys->min_incl; + + i = 0; + foreach(lc, old_keys->maxkeys) + { + Node *val = lfirst(lc); + Node *n = eval_const_expressions_from_list(estate->es_param_list_info, val); + + keys->maxkeys_datums[i++] = ((Const *) n)->constvalue; + } + keys->n_maxkeys = old_keys->n_maxkeys; + keys->max_incl = old_keys->max_incl; + + memcpy(keys->keynullness, old_keys->keynullness, sizeof(keys->keynullness)); + + return keys; +} + + +/* ---------------------------------------------------------------- + * exec_InitExpr + * + * Evaluate and store the ooutput of ExecInitExpr for each of the keys. + * ---------------------------------------------------------------- + */ +static PartScanKeyInfo * +exec_InitExpr(PlanState *parent, PartScanKeyInfo *old_keys) +{ + PartScanKeyInfo *keys; + ListCell *lc; + + if (old_keys == NULL) + return NULL; + + keys = makeNode(PartScanKeyInfo); + + foreach(lc, old_keys->eqkeys) + { + Expr *val = (Expr *) lfirst(lc); + + keys->eqkeys = lappend(keys->eqkeys, ExecInitExpr(val, parent)); + } + keys->n_eqkeys = old_keys->n_eqkeys; + + foreach(lc, old_keys->minkeys) + { + Expr *val = (Expr *) lfirst(lc); + + keys->minkeys = lappend(keys->minkeys, ExecInitExpr(val, parent)); + } + keys->n_minkeys = old_keys->n_minkeys; + keys->min_incl = old_keys->min_incl; + + foreach(lc, old_keys->maxkeys) + { + Expr *val = (Expr *) lfirst(lc); + + keys->maxkeys = lappend(keys->maxkeys, ExecInitExpr(val, parent)); + } + keys->n_maxkeys = old_keys->n_maxkeys; + keys->max_incl = old_keys->max_incl; + + memcpy(keys->keynullness, old_keys->keynullness, sizeof(keys->keynullness)); + return keys; +} + +/* ---------------------------------------------------------------- * ExecInitAppend * * Begin all of the subscans of the append node. @@ -165,22 +270,97 @@ ExecInitAppend(Append *node, EState *estate, int eflags) */ ExecInitResultTupleSlot(estate, &appendstate->ps); - /* - * call ExecInitNode on each of the plans to be executed and save the - * results into the array "appendplans". - */ - i = 0; - foreach(lc, node->appendplans) + if (node->et_keys) { - Plan *initNode = (Plan *) lfirst(lc); + List *temp_appendplans = NIL; + Append *temp_node; + Relation rel; + PartScanKeyInfo *keys; + PartitionSet *partset; + int i; + + keys = extern_eval_const_expressions(estate, node->et_keys); + + rel = relation_open(node->parentoid, NoLock); + partset = get_partitions_for_keys(rel, keys); + relation_close(rel, NoLock); - appendplanstates[i] = ExecInitNode(initNode, estate, eflags); - i++; + if (!partset->empty) + { + if (partset->all_parts) + { + i = 0; + foreach(lc, node->appendplans) + { + Plan *initNode = (Plan *) lfirst(lc); + + temp_appendplans = lappend(temp_appendplans, initNode); + + appendplanstates[i] = ExecInitNode(initNode, estate, eflags); + i++; + } + } + else + { + int j; + + i = 0; + if (partset->use_range) + { + for (j = partset->min_part_idx; j <= partset->max_part_idx; j++) + { + Plan *initNode = (Plan *) list_nth(node->appendplans, j); + + temp_appendplans = lappend(temp_appendplans, initNode); + appendplanstates[i++] = ExecInitNode(initNode, estate, eflags); + } + } + + if (!bms_is_empty(partset->other_parts)) + { + while ((j = bms_first_member(partset->other_parts)) >= 0) + { + Plan *initNode = (Plan *) list_nth(node->appendplans, j); + + temp_appendplans = lappend(temp_appendplans, initNode); + appendplanstates[i++] = ExecInitNode(initNode, estate, eflags); + } + } + } + } + /* create new AppendState for our append node */ + temp_node = copyObject(node); + temp_node->appendplans = temp_appendplans; + ((Plan *) temp_node)->plan_rows = i; + appendstate->as_nplans = i; + appendstate->ps.plan = (Plan *) temp_node; + appendstate->ps.state = estate; + appendstate->ps.ExecProcNode = ExecAppend; + appendstate->appendplans = (PlanState **) palloc0(i * sizeof(PlanState *)); + appendstate->appendplans = appendplanstates; } + else + { + appendstate->parentoid = node->parentoid; + ExecAssignExprContext(estate, &appendstate->ps); + + /* + * call ExecInitNode on each of the plans to be executed and save the + * results into the array "appendplans". + */ + i = 0; + foreach(lc, node->appendplans) + { + Plan *initNode = (Plan *) lfirst(lc); + + appendplanstates[i] = ExecInitNode(initNode, estate, eflags); + i++; + } + } + + if (node->ex_keys) + appendstate->ex_keys = exec_InitExpr((PlanState *) appendstate, node->ex_keys); - /* - * initialize output tuple type - */ ExecAssignResultTypeFromTL(&appendstate->ps); appendstate->ps.ps_ProjInfo = NULL; @@ -237,7 +417,21 @@ ExecAppend(PlanState *pstate) * ExecInitAppend. */ if (ScanDirectionIsForward(node->ps.state->es_direction)) - node->as_whichplan++; + { + /* + * For runtime partition pruning, goto the next valid partition + * index + */ + if (node->index) + { + if (++node->as_whichpartition < list_length(node->index)) + node->as_whichplan = list_nth_int(node->index, node->as_whichpartition); + + return ExecClearTuple(node->ps.ps_ResultTupleSlot); + } + else + node->as_whichplan++; + } else node->as_whichplan--; if (!exec_append_initialize_next(node)) @@ -275,11 +469,95 @@ ExecEndAppend(AppendState *node) ExecEndNode(appendplans[i]); } +/* ---------------------------------------------------------------- + * exec_EvalExpr + * + * Convert the expressions to a constant. + * ---------------------------------------------------------------- + */ +static PartScanKeyInfo* +exec_EvalExpr(ExprContext *econtext, PartScanKeyInfo *old_keys) +{ + PartScanKeyInfo *keys; + int i; + bool isnull; + ListCell *lc; + + if (old_keys == NULL) + return NULL; + + keys = makeNode(PartScanKeyInfo); + i = 0; + foreach(lc, old_keys->eqkeys) + { + ExprState *val = lfirst(lc); + + keys->eqkeys_datums[i++] = ExecEvalExpr(val, econtext, &isnull); + } + keys->n_eqkeys = old_keys->n_eqkeys; + + i = 0; + foreach(lc, old_keys->minkeys) + { + ExprState *val = lfirst(lc); + + keys->minkeys_datums[i++] = ExecEvalExpr(val, econtext, &isnull); + } + keys->n_minkeys = old_keys->n_minkeys; + keys->min_incl = old_keys->min_incl; + + i = 0; + foreach(lc, old_keys->maxkeys) + { + ExprState *val = lfirst(lc); + + keys->maxkeys_datums[i++] = ExecEvalExpr(val, econtext, &isnull); + } + keys->n_maxkeys = old_keys->n_maxkeys; + keys->max_incl = old_keys->max_incl; + + memcpy(keys->keynullness, old_keys->keynullness, sizeof(keys->keynullness)); + return keys; +} + void ExecReScanAppend(AppendState *node) { int i; + if (node->ps.chgParam != NULL && node->ex_keys) + { + PartitionSet *partset; + + node->index = NULL; + node->ex_keys = exec_EvalExpr(node->ps.ps_ExprContext, node->ex_keys); + if (node->parentoid) + { + Relation rel = relation_open(node->parentoid, NoLock); + + partset = get_partitions_for_keys(rel, node->ex_keys); + relation_close(rel, NoLock); + } + if (!partset->empty) + { + if (!partset->all_parts) + { + int j; + + i = 0; + if (partset->use_range) + { + for (j = partset->min_part_idx; j <= partset->max_part_idx; j++) + node->index = lappend_int(node->index, j); + } + if (!bms_is_empty(partset->other_parts)) + { + while ((j = bms_first_member(partset->other_parts)) >= 0) + node->index = lappend_int(node->index, j); + } + } + } + } for (i = 0; i < node->as_nplans; i++) { PlanState *subnode = node->appendplans[i]; @@ -298,6 +576,13 @@ ExecReScanAppend(AppendState *node) if (subnode->chgParam == NULL) ExecReScan(subnode); } - node->as_whichplan = 0; + + if (node->index) + { + node->as_whichplan = linitial_int(node->index); + node->as_whichpartition = 0; + } + else + node->as_whichplan = 0; exec_append_initialize_next(node); } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index c1a83ca..95dddab 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -224,6 +224,27 @@ _copyModifyTable(const ModifyTable *from) return newnode; } +static PartScanKeyInfo * +_copyPartScanKeyInfo(const PartScanKeyInfo *from) +{ + PartScanKeyInfo *newnode = makeNode(PartScanKeyInfo); + + COPY_SCALAR_FIELD(n_eqkeys); + COPY_NODE_FIELD(eqkeys); + + COPY_SCALAR_FIELD(n_minkeys); + COPY_NODE_FIELD(minkeys); + COPY_SCALAR_FIELD(min_incl); + + COPY_SCALAR_FIELD(n_maxkeys); + COPY_NODE_FIELD(maxkeys); + COPY_SCALAR_FIELD(max_incl); + + memcpy(newnode->keynullness, from->keynullness, sizeof(newnode->keynullness)); + + return newnode; +} + /* * _copyAppend */ @@ -242,6 +263,9 @@ _copyAppend(const Append *from) */ COPY_NODE_FIELD(partitioned_rels); COPY_NODE_FIELD(appendplans); + COPY_NODE_FIELD(ex_keys); + COPY_NODE_FIELD(et_keys); + COPY_SCALAR_FIELD(parentoid); return newnode; } @@ -5014,6 +5038,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; + case T_PartScanKeyInfo: + retval = _copyPartScanKeyInfo(from); + break; /* * VALUE NODES diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index ccb6a1f..15e6764 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -2652,6 +2652,8 @@ parseNodeString(void) return_value = _readLimit(); else if (MATCH("NESTLOOPPARAM", 13)) return_value = _readNestLoopParam(); + else if (MATCH("PARTSCANKEYINFO", 15)) + return_value = _readNestLoopParam(); else if (MATCH("PLANROWMARK", 11)) return_value = _readPlanRowMark(); else if (MATCH("PLANINVALITEM", 13)) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 12f409e..2d134c7 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -18,6 +18,7 @@ #include #include +#include "catalog/partition.h" #include "access/sysattr.h" #include "access/tsmapi.h" #include "catalog/partition.h" @@ -139,11 +140,11 @@ static void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); static List *get_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte); + RangeTblEntry *rte, bool is_runtime); static List *match_clauses_to_partkey(RelOptInfo *rel, List *clauses, bool *contains_const, - bool *constfalse); + bool *constfalse, bool is_runtime); /* @@ -865,6 +866,45 @@ intcmp(const void *va, const void *vb) return (a > b) ? 1 : -1; } +PartScanKeyInfo* +get_append_partition_info(RelOptInfo *rel, RangeTblEntry *rte, + bool is_runtime, List *clauses) +{ + bool constfalse, + contains_constant; + PartScanKeyInfo *keys = NULL; + List *partclauses; + + /* + * Get the clauses that match the partition key, including information + * about any nullness tests against partition keys. Set keynullness to a + * invalid value of NullTestType, which 0 is not. + */ + partclauses = match_clauses_to_partkey(rel, + list_copy(clauses), + &contains_constant, + &constfalse, is_runtime); + + if (partclauses != NIL) + { + bool constfalse; + List *or_clauses; + Relation parent = heap_open(rte->relid, NoLock); + PartScanKeyInfo *temp_keys = makeNode(PartScanKeyInfo); + int nkeys = classify_partition_bounding_keys(parent, partclauses, + temp_keys, + &constfalse, + &or_clauses, + true); + + if (nkeys > 0) + keys = temp_keys; + + heap_close(parent, NoLock); + } + return keys; +} + /* * get_append_rel_partitions * Return the list of partitions of rel that pass the clauses mentioned @@ -875,7 +915,7 @@ intcmp(const void *va, const void *vb) static List * get_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel, - RangeTblEntry *rte) + RangeTblEntry *rte, bool is_runtime) { Relation parent = heap_open(rte->relid, NoLock); PartitionDesc partdesc = RelationGetPartitionDesc(parent); @@ -898,7 +938,7 @@ get_append_rel_partitions(PlannerInfo *root, partclauses = match_clauses_to_partkey(rel, list_copy(rel->baserestrictinfo), &contains_const, - &constfalse); + &constfalse, is_runtime); /* * If the matched clauses contains at least some constant operands, use @@ -1004,7 +1044,8 @@ static List * match_clauses_to_partkey(RelOptInfo *rel, List *clauses, bool *contains_const, - bool *constfalse) + bool *constfalse, + bool is_runtime) { PartitionScheme partscheme = rel->part_scheme; List *result = NIL; @@ -1051,7 +1092,8 @@ match_clauses_to_partkey(RelOptInfo *rel, match_clauses_to_partkey(rel, list_copy(((BoolExpr *) clause)->args), &contains_const1, - &constfalse1) != NIL) + &constfalse1, + is_runtime) != NIL) { result = lappend(result, clause); *contains_const = contains_const1; @@ -1123,6 +1165,12 @@ match_clauses_to_partkey(RelOptInfo *rel, /* Neither argument matches the partition key. */ continue; + if (IsA(constexpr, Const) &&is_runtime) + continue; + + if (IsA(constexpr, Param) &&!is_runtime) + continue; + /* * Only allow strict operators to think sanely about the * behavior with null arguments. @@ -1228,7 +1276,7 @@ match_clauses_to_partkey(RelOptInfo *rel, if (!PartCollMatchesExprColl(partcoll, saop_coll)) continue; } - else if (IsA(clause, NullTest)) + else if (!is_runtime && IsA(clause, NullTest)) { NullTest *nulltest = (NullTest *) clause; Node *arg = (Node *) nulltest->arg; @@ -1288,7 +1336,7 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, } else { - rel_appinfos = get_append_rel_partitions(root, rel, rte); + rel_appinfos = get_append_rel_partitions(root, rel, rte, false); rel->live_partitioned_rels = list_make1_int(rti); } @@ -1850,7 +1898,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, * if we have zero or one live subpath due to constraint exclusion.) */ if (subpaths_valid) - add_path(rel, (Path *) create_append_path(rel, subpaths, NULL, 0, + add_path(rel, (Path *) create_append_path(root, rel, subpaths, NULL, 0, partitioned_rels)); /* @@ -1877,7 +1925,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, Assert(parallel_workers > 0); /* Generate a partial append path. */ - appendpath = create_append_path(rel, partial_subpaths, NULL, + appendpath = create_append_path(root, rel, partial_subpaths, NULL, parallel_workers, partitioned_rels); add_partial_path(rel, (Path *) appendpath); } @@ -1931,7 +1979,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (subpaths_valid) add_path(rel, (Path *) - create_append_path(rel, subpaths, required_outer, 0, + create_append_path(root, rel, subpaths, required_outer, 0, partitioned_rels)); } } @@ -2168,7 +2216,7 @@ set_dummy_rel_pathlist(RelOptInfo *rel) rel->pathlist = NIL; rel->partial_pathlist = NIL; - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NULL, 0, NIL)); /* * We set the cheapest path immediately, to ensure that IS_DUMMY_REL() diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 7356683..af7e790 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -1232,7 +1232,7 @@ mark_dummy_rel(RelOptInfo *rel) rel->partial_pathlist = NIL; /* Set up the dummy path */ - add_path(rel, (Path *) create_append_path(rel, NIL, NULL, 0, NIL)); + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NULL, 0, NIL)); /* Set or update cheapest_total_path and related fields */ set_cheapest(rel); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 9c74e39..9014ef2 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1001,6 +1001,47 @@ create_join_plan(PlannerInfo *root, JoinPath *best_path) return plan; } +static PartScanKeyInfo * +replace_partition_nestloop_params(PlannerInfo *root, PartScanKeyInfo *old_keys) +{ + PartScanKeyInfo *keys; + ListCell *lc; + + if (old_keys == NULL) + return NULL; + + keys = makeNode(PartScanKeyInfo); + foreach(lc, old_keys->eqkeys) + { + Node *n = lfirst(lc); + + keys->eqkeys = lappend(keys->eqkeys, replace_nestloop_params(root, n)); + } + keys->n_eqkeys = old_keys->n_eqkeys; + + foreach(lc, old_keys->minkeys) + { + Node *n = lfirst(lc); + + keys->minkeys = lappend(keys->minkeys, replace_nestloop_params(root, n)); + } + keys->n_minkeys = old_keys->n_minkeys; + keys->min_incl = old_keys->min_incl; + + foreach(lc, old_keys->maxkeys) + { + Node *n = lfirst(lc); + + keys->maxkeys = lappend(keys->maxkeys, replace_nestloop_params(root, n)); + } + keys->n_maxkeys = old_keys->n_maxkeys; + keys->max_incl = old_keys->max_incl; + + memcpy(keys->keynullness, old_keys->keynullness, sizeof(keys->keynullness)); + return keys; +} + + /* * create_append_plan * Create an Append plan for 'best_path' and (recursively) plans @@ -1063,6 +1104,11 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path) copy_generic_path_info(&plan->plan, (Path *) best_path); + + plan->ex_keys = replace_partition_nestloop_params(root, best_path->ex_keys); + plan->et_keys = best_path->et_keys; + plan->parentoid = best_path->parentoid; + return (Plan *) plan; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index fd0e483..8a8876f 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3675,7 +3675,7 @@ create_grouping_paths(PlannerInfo *root, paths = lappend(paths, path); } path = (Path *) - create_append_path(grouped_rel, + create_append_path(root, grouped_rel, paths, NULL, 0, diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index f620243..12d0f85 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -590,7 +590,7 @@ generate_union_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NULL, 0, NIL); + path = (Path *) create_append_path(root, result_rel, pathlist, NULL, 0, NIL); /* We have to manually jam the right tlist into the path; ick */ path->pathtarget = create_pathtarget(root, tlist); @@ -702,7 +702,7 @@ generate_nonunion_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NULL, 0, NIL); + path = (Path *) create_append_path(root, result_rel, pathlist, NULL, 0, NIL); /* We have to manually jam the right tlist into the path; ick */ path->pathtarget = create_pathtarget(root, tlist); diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 30cdd3d..8634460 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -5197,3 +5197,19 @@ tlist_matches_coltypelist(List *tlist, List *coltypelist) return true; } + +Node * +eval_const_expressions_from_list(ParamListInfo prmList, Node *node) +{ + eval_const_expressions_context context; + + if (prmList) + context.boundParams = prmList; /* bound Params */ + else + context.boundParams = NULL; + context.root = NULL; + context.active_fns = NIL; /* nothing being recursively simplified */ + context.case_val = NULL; /* no CASE being examined */ + context.estimate = false; /* safe transformations only */ + return eval_const_expressions_mutator(node, &context); +} diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 36ec025..8910d39 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1208,7 +1208,7 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, * Note that we must handle subpaths = NIL, representing a dummy access path. */ AppendPath * -create_append_path(RelOptInfo *rel, List *subpaths, Relids required_outer, +create_append_path(PlannerInfo *root, RelOptInfo *rel, List *subpaths, Relids required_outer, int parallel_workers, List *partitioned_rels) { AppendPath *pathnode = makeNode(AppendPath); @@ -1253,6 +1253,40 @@ create_append_path(RelOptInfo *rel, List *subpaths, Relids required_outer, Assert(bms_equal(PATH_REQ_OUTER(subpath), required_outer)); } + if (root && (required_outer || rel->baserestrictinfo)) + { + RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); + List *query_quals = rel->baserestrictinfo; + + if (rte && rte->rtekind == RTE_RELATION) + { + Oid poid = rte->relid; + Relation prel = relation_open(poid, NoLock); + + if (prel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + ParamPathInfo *ppi = pathnode->path.param_info; + + /* Set the ex_keys for Exec Params */ + if (ppi) + { + List *ppi_clauses = ppi->ppi_clauses; + + pathnode->ex_keys = + get_append_partition_info(rel, rte, true, ppi_clauses); + } + /* Set et_keys for extern params */ + if (query_quals) + { + pathnode->et_keys = + get_append_partition_info(rel, rte, true, query_quals); + } + pathnode->parentoid = poid; + } + relation_close(prel, NoLock); + } + } + return pathnode; } diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index b06696b..ca83bb8 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -1556,6 +1556,9 @@ ParamPathInfo * get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) { ParamPathInfo *ppi; + Relids joinrelids; + List *pclauses; + ListCell *lc; /* Unparameterized paths have no ParamPathInfo */ if (bms_is_empty(required_outer)) @@ -1567,11 +1570,29 @@ get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) if ((ppi = find_param_path_info(appendrel, required_outer))) return ppi; + + /* + * Generally for appendrel we don't fetch the clause from the the + * join clause (only we do so for baserel), but for identifying whether + * the appendrel is applicable for runtime pruning or not. + */ + joinrelids = bms_union(appendrel->relids, required_outer); + pclauses = NIL; + foreach(lc, appendrel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + if (join_clause_is_movable_into(rinfo, + appendrel->relids, + joinrelids)) + pclauses = lappend(pclauses, rinfo); + } + /* Else build the ParamPathInfo */ ppi = makeNode(ParamPathInfo); ppi->ppi_req_outer = required_outer; ppi->ppi_rows = 0; - ppi->ppi_clauses = NIL; + ppi->ppi_clauses = pclauses; appendrel->ppilist = lappend(appendrel->ppilist, ppi); return ppi; diff --git a/src/include/catalog/partition.h b/src/include/catalog/partition.h index 7da99a9..eb54eaf 100644 --- a/src/include/catalog/partition.h +++ b/src/include/catalog/partition.h @@ -16,6 +16,7 @@ #include "fmgr.h" #include "executor/tuptable.h" #include "nodes/execnodes.h" +#include "nodes/relation.h" #include "parser/parse_node.h" #include "utils/rel.h" @@ -112,4 +113,11 @@ extern List *get_proposed_default_constraint(List *new_part_constaints); void get_partitions_from_clauses(Relation relation, List *partclauses, int *min_part_idx, int *max_part_idx, Bitmapset **other_parts); + + PartitionSet *get_partitions_for_keys(Relation rel, + PartScanKeyInfo *keys); + +extern int classify_partition_bounding_keys(Relation relation, List *clauses, + PartScanKeyInfo* keys, bool *constfalse, + List **or_clauses, bool is_runtime); #endif /* PARTITION_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e05bc04..60cbb36 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1006,6 +1006,11 @@ typedef struct AppendState PlanState **appendplans; /* array of PlanStates for my inputs */ int as_nplans; int as_whichplan; + Oid parentoid; + List *index; + + int as_whichpartition; + PartScanKeyInfo *ex_keys; } AppendState; /* ---------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index ffeeb49..5e09258 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -87,6 +87,7 @@ typedef enum NodeTag T_NestLoopParam, T_PlanRowMark, T_PlanInvalItem, + T_PartScanKeyInfo, /* * TAGS FOR PLAN STATE NODES (execnodes.h) diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index dd74efa..96f0632 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -19,6 +19,7 @@ #include "nodes/bitmapset.h" #include "nodes/lockoptions.h" #include "nodes/primnodes.h" +#include "nodes/relation.h" /* ---------------------------------------------------------------- @@ -248,6 +249,9 @@ typedef struct Append /* RT indexes of non-leaf tables in a partition tree */ List *partitioned_rels; List *appendplans; + Oid parentoid; + PartScanKeyInfo *ex_keys; + PartScanKeyInfo *et_keys; } Append; /* ---------------- diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 9c67bd1..ee4f6de 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -363,6 +363,89 @@ typedef struct PartitionSchemeData typedef struct PartitionSchemeData *PartitionScheme; +/* + * PartScanKeyInfo + * Bounding scan keys to look up a table's partitions obtained from + * mutually-ANDed clauses containing partitioning-compatible operators + */ +typedef struct PartScanKeyInfo +{ + NodeTag type; + + /* + * Expressions or constants constituting the *whole* partition key + * compared using partitioning-compatible equality operator(s). When + * n_eqkeys > 0, other keys (minkeys and maxkeys) are irrelevant. + */ + List *eqkeys; + Datum eqkeys_datums[PARTITION_MAX_KEYS]; + int n_eqkeys; + + /* + * Expressions or constants that constitute the lower bound on the + * partition key or a prefix thereof. The last of those constants is + * compared using > or >= operator compatible with partitioning, making + * this the lower bound in a range query. + */ + List *minkeys; + Datum minkeys_datums[PARTITION_MAX_KEYS]; + int n_minkeys; + bool min_incl; + + /* + * Expressions or constants that constitute the upper bound on the + * partition key or a prefix thereof. The last of those constants is + * compared using < or <= operator compatible with partitioning, making + * this the upper bound in a range query. + */ + List *maxkeys; + Datum maxkeys_datums[PARTITION_MAX_KEYS]; + int n_maxkeys; + bool max_incl; + + /* + * Specifies the type of NullTest that was applied to each of the + * partition key columns or -1 if none was applied. Partitioning handles + * null partition keys specially depending on the partitioning method in + * use, so get_partitions_for_keys can return partitions according to the + * nullness condition for partition keys. + */ + NullTestType keynullness[PARTITION_MAX_KEYS]; +} PartScanKeyInfo; + + /* A data structure to represent a partition set. */ +typedef struct PartitionSet +{ + /* + * If either empty or all_parts is true, values of the other fields are + * invalid. + */ + bool empty; /* contains no partitions */ + bool all_parts; /* contains all partitions */ + + /* + * In the case of range partitioning, min_part_index contains the index of + * the lowest partition contained in the set and max_datum_index that of + * the highest partition (all partitions between these two indexes + * inclusive are part of the set.) Since other types of partitioning do + * not impose order on the data contained in successive partitions, these + * fields are not set in that case. + */ + bool use_range; + int min_part_idx; + int max_part_idx; + + /* + * other_parts contains the indexes of partitions that are not covered by + * the range defined by min/max indexes. For example, in the case of + * range partitoning, it will include default partition index (if any). + * Also, this is the only way to return list partitions, because list + * partitions do not have the same ordering property as range partitions, + * so it's pointless to use the min/max range method. + */ + Bitmapset *other_parts; +} PartitionSet; + /*---------- * RelOptInfo * Per-relation information for planning/optimization @@ -1289,6 +1372,9 @@ typedef struct AppendPath /* RT indexes of non-leaf tables in a partition tree */ List *partitioned_rels; List *subpaths; /* list of component Paths */ + PartScanKeyInfo *ex_keys; /* Runtime Partition Pruning - EXEC Params */ + PartScanKeyInfo *et_keys; /* Runtime Partition Pruning - EXTERN Params */ + Oid parentoid; } AppendPath; #define IS_DUMMY_PATH(p) \ @@ -2334,4 +2420,5 @@ typedef struct JoinCostWorkspace int numbatches; } JoinCostWorkspace; + #endif /* RELATION_H */ diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index e367221..c19ee31 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -79,6 +79,7 @@ extern void CommuteOpExpr(OpExpr *clause); extern void CommuteRowCompareExpr(RowCompareExpr *clause); extern Node *eval_const_expressions(PlannerInfo *root, Node *node); +extern Node *eval_const_expressions_from_list(ParamListInfo prm_list, Node *node); extern Node *estimate_expression_value(PlannerInfo *root, Node *node); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index c1f2fc9..d9b8dff 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -63,7 +63,7 @@ extern BitmapOrPath *create_bitmap_or_path(PlannerInfo *root, List *bitmapquals); extern TidPath *create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, Relids required_outer); -extern AppendPath *create_append_path(RelOptInfo *rel, List *subpaths, +extern AppendPath *create_append_path(PlannerInfo *root, RelOptInfo *rel, List *subpaths, Relids required_outer, int parallel_workers, List *partitioned_rels); extern MergeAppendPath *create_merge_append_path(PlannerInfo *root, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index ea886b6..e780b20 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -234,4 +234,8 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, EquivalenceClass *eclass, Oid opfamily, int strategy, bool nulls_first); +extern PartScanKeyInfo *get_append_partition_info( + RelOptInfo *rel, + RangeTblEntry *rte, bool is_runtime, List *clauses); + #endif /* PATHS_H */ -- 1.8.3.1