diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c index 7e3a777..e0c818a 100644 --- a/src/backend/catalog/partition.c +++ b/src/backend/catalog/partition.c @@ -34,6 +34,7 @@ #include "catalog/pg_type.h" #include "commands/tablecmds.h" #include "executor/executor.h" +#include "executor/nodeSubplan.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -279,11 +280,14 @@ static uint64 compute_hash_value(PartitionKey key, Datum *values, bool *isnull); PG_FUNCTION_INFO_V1(satisfies_hash_partition); static Bitmapset *get_partitions_from_clauses_recurse(Relation relation, - int rt_index, List *clauses); + int rt_index, List *clauses, + ParamListInfo base_prmlist, + ExprContext *econtext); static int classify_partition_bounding_keys(Relation relation, List *clauses, int rt_index, PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses); + List **or_clauses, ParamListInfo base_prmlist, + ExprContext *econtext); static void remove_redundant_clauses(PartitionKey partkey, int partattoff, List *all_clauses, List **result, bool *constfalse); @@ -293,7 +297,8 @@ static bool partition_cmp_args(PartitionKey key, int partattoff, static int32 partition_op_strategy(PartitionKey key, PartClause *op, bool *incl); static bool partkey_datum_from_expr(PartitionKey key, int partattoff, - Expr *expr, Datum *value); + Expr *expr, Datum *value, ParamListInfo base_prmlist, + ExprContext *econtext); static Bitmapset *get_partitions_for_keys(Relation rel, PartScanKeyInfo *keys); @@ -1663,6 +1668,37 @@ get_partition_qual_relid(Oid relid) return result; } +/* get_leaf_part_recurse + * Get the leaf oids for the given rel. + */ +void +get_leaf_part_recurse(Relation rel, List **leaf_part_oids) +{ + PartitionDesc partdesc = RelationGetPartitionDesc(rel); + int i; + + check_stack_depth(); + + for (i = 0; i < partdesc->nparts; i++) + { + Oid partrelid = partdesc->oids[i]; + + if (get_rel_relkind(partrelid) != RELKIND_PARTITIONED_TABLE) + *leaf_part_oids = lappend_oid(*leaf_part_oids, partrelid); + else + { + /* + * We assume all tables in the partition tree were already locked + * by the caller. + */ + Relation partrel = heap_open(partrelid, NoLock); + + get_leaf_part_recurse(partrel, leaf_part_oids); + heap_close(partrel, NoLock); + } + } +} + /* * get_partitions_from_clauses * Determine the set of partitions of relation that will satisfy all @@ -1673,7 +1709,8 @@ get_partition_qual_relid(Oid relid) */ Bitmapset * get_partitions_from_clauses(Relation relation, int rt_index, - List *partclauses) + List *partclauses, ParamListInfo base_prmlist, + ExprContext *econtext) { Bitmapset *result; List *partconstr = RelationGetPartitionQual(relation); @@ -1707,7 +1744,8 @@ get_partitions_from_clauses(Relation relation, int rt_index, } result = get_partitions_from_clauses_recurse(relation, rt_index, - partclauses); + partclauses, base_prmlist, + econtext); return result; } @@ -1723,7 +1761,8 @@ get_partitions_from_clauses(Relation relation, int rt_index, */ static Bitmapset * get_partitions_from_clauses_recurse(Relation relation, int rt_index, - List *clauses) + List *clauses, ParamListInfo base_prmlist, + ExprContext *econtext) { PartitionDesc partdesc = RelationGetPartitionDesc(relation); Bitmapset *result = NULL; @@ -1739,7 +1778,8 @@ get_partitions_from_clauses_recurse(Relation relation, int rt_index, */ nkeys = classify_partition_bounding_keys(relation, clauses, rt_index, &keys, &constfalse, - &or_clauses); + &or_clauses, base_prmlist, + econtext); /* * classify_partition_bounding_keys() may have found clauses marked @@ -1799,7 +1839,9 @@ get_partitions_from_clauses_recurse(Relation relation, int rt_index, arg_partset = get_partitions_from_clauses_recurse(relation, rt_index, - arg_clauses); + arg_clauses, + base_prmlist, + econtext); /* * Partition sets obtained from mutually-disjunctive clauses are @@ -1857,7 +1899,8 @@ static int classify_partition_bounding_keys(Relation relation, List *clauses, int rt_index, PartScanKeyInfo *keys, bool *constfalse, - List **or_clauses) + List **or_clauses, ParamListInfo base_prmlist, + ExprContext *econtext) { PartitionKey partkey = RelationGetPartitionKey(relation); int i; @@ -1897,6 +1940,9 @@ classify_partition_bounding_keys(Relation relation, List *clauses, continue; } } + /* when called from ExecReScanAppend */ + else if (IsA(lfirst(lc), ExprState)) + clause = ((ExprState *) lfirst(lc))->expr; else clause = (Expr *) lfirst(lc); @@ -2332,7 +2378,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, if (op_strategy < 0 && need_next_max && partkey_datum_from_expr(partkey, i, constarg, - &keys->maxkeys[i])) + &keys->maxkeys[i], base_prmlist, + econtext)) { keys->n_maxkeys++; keys->max_incl = incl; @@ -2344,12 +2391,14 @@ classify_partition_bounding_keys(Relation relation, List *clauses, Assert(incl); if (need_next_eq && partkey_datum_from_expr(partkey, i, constarg, - &keys->eqkeys[i])) + &keys->eqkeys[i], base_prmlist, + econtext)) keys->n_eqkeys++; if (need_next_max && partkey_datum_from_expr(partkey, i, constarg, - &keys->maxkeys[i])) + &keys->maxkeys[i], base_prmlist, + econtext)) { keys->n_maxkeys++; keys->max_incl = true; @@ -2357,7 +2406,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, if (need_next_min && partkey_datum_from_expr(partkey, i, constarg, - &keys->minkeys[i])) + &keys->minkeys[i], base_prmlist, + econtext)) { keys->n_minkeys++; keys->min_incl = true; @@ -2365,7 +2415,8 @@ classify_partition_bounding_keys(Relation relation, List *clauses, } else if (need_next_min && partkey_datum_from_expr(partkey, i, constarg, - &keys->minkeys[i])) + &keys->minkeys[i], base_prmlist, + econtext)) { keys->n_minkeys++; keys->min_incl = incl; @@ -2453,7 +2504,8 @@ partition_op_strategy(PartitionKey key, PartClause *op, bool *incl) */ static bool partkey_datum_from_expr(PartitionKey key, int partattoff, - Expr *expr, Datum *value) + Expr *expr, Datum *value, ParamListInfo base_prmlist, + ExprContext *econtext) { Oid exprtype = exprType((Node *) expr); @@ -2496,6 +2548,46 @@ partkey_datum_from_expr(PartitionKey key, int partattoff, *value = ((Const *) expr)->constvalue; return true; + case T_Param: + switch (((Param *) expr)->paramkind) + { + case PARAM_EXTERN: + if (base_prmlist) + { + Node *n; + + n = eval_const_expressions_from_list(base_prmlist, + (Node *) expr); + + if (IsA(n, Const)) + { + *value = ((Const *) n)->constvalue; + return true; + } + } + return false; + + case PARAM_EXEC: + if (econtext) + { + Param *param = (Param *) expr; + ParamExecData *prm; + + prm = &(econtext->ecxt_param_exec_vals[param->paramid]); + if (unlikely(prm->execPlan != NULL)) + { + ExecSetParamPlan(prm->execPlan, econtext); + Assert(prm->execPlan == NULL); + } + *value = prm->value; + return true; + } + + default: + return false; + } + return false; + default: return false; } @@ -2758,10 +2850,12 @@ partition_cmp_args(PartitionKey key, int partattoff, Assert(op->valid_cache && leftarg->valid_cache && rightarg->valid_cache); /* Get the constant values from the operands */ if (!partkey_datum_from_expr(key, partattoff, - leftarg->constarg, &leftarg_const)) + leftarg->constarg, &leftarg_const, + NULL, NULL)) return false; if (!partkey_datum_from_expr(key, partattoff, - rightarg->constarg, &rightarg_const)) + rightarg->constarg, &rightarg_const, + NULL, NULL)) return false; /* diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 7e4fbaf..e3ac4ac 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -1347,6 +1347,21 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoChar(es->str, '\n'); + /* run-time pruning information for Append node */ + if (es->analyze && IsA(plan, Append)) + { + Append *append = (Append *) plan; + + if (append->base_params) + show_scan_qual(append->base_params, "Runtime Partition Pruning", + planstate, ancestors, es); + + if (append->join_clauses) + show_scan_qual(append->join_clauses, + "Runtime Partition Pruning Join Filter", + planstate, ancestors, es); + } + /* target list */ if (es->verbose) show_plan_tlist(planstate, ancestors, es); diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index d545af2..b6872a2 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -23,8 +23,6 @@ #include "utils/rls.h" #include "utils/ruleutils.h" -static PartitionDispatch *RelationGetPartitionDispatchInfo(Relation rel, - int *num_parted, List **leaf_part_oids); static void get_partition_dispatch_recurse(Relation rel, Relation parent, List **pds, List **leaf_part_oids); static void FormPartitionKeyDatum(PartitionDispatch pd, @@ -284,7 +282,7 @@ ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, * All the relations in the partition tree (including 'rel') must have been * locked (using at least the AccessShareLock) by the caller. */ -static PartitionDispatch * +PartitionDispatch * RelationGetPartitionDispatchInfo(Relation rel, int *num_parted, List **leaf_part_oids) { diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 0e93713..d089fc3 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -57,15 +57,20 @@ #include "postgres.h" +#include "catalog/pg_inherits_fn.h" #include "executor/execdebug.h" +#include "executor/execPartition.h" #include "executor/nodeAppend.h" #include "miscadmin.h" +#include /* Shared state for parallel-aware Append. */ struct ParallelAppendState { LWLock pa_lock; /* mutual exclusion to choose next subplan */ + LWLock prune_lock; /* mutual exclusion to choose next subplan */ int pa_next_plan; /* next plan to choose by any worker */ + bool is_runtime_pruned; /* * pa_finished[i] should be true if no more workers should select subplan @@ -84,6 +89,85 @@ static bool choose_next_subplan_for_leader(AppendState *node); static bool choose_next_subplan_for_worker(AppendState *node); /* ---------------------------------------------------------------- + * set_append_subplan_indexes + * + * Determine the subplans to scan based on the clauses. + * ---------------------------------------------------------------- + */ +static void +set_append_subplan_indexes(PlanState *pstate, AppendState *node, List *clauses) +{ + ParallelAppendState *as_pstate = node->as_pstate; + Append *plan = (Append *) pstate->plan; + Bitmapset *partset = NULL, + *subplans = NULL; + PartitionDispatch *pd = (PartitionDispatch *) plan->pd, + parent, + *p1; + List *parents = NIL; + int cur_index, + i; + + parents = lappend(parents, &pd[0]); + node->subplan_indexes = NIL; + do + { + p1 = linitial(parents); + parent = *p1; + + partset = get_partitions_from_clauses(parent->reldesc, + 1, + list_copy(clauses), + pstate->state->es_param_list_info, + node->ps.ps_ExprContext); + + if (!bms_is_empty(partset)) + { + while ((cur_index = bms_first_member(partset)) >= 0) + { + if (parent->indexes[cur_index] >= 0) + subplans = + bms_add_member(subplans, parent->indexes[cur_index]); + else + parents = + lappend(parents, &pd[-parent->indexes[cur_index]]); + } + } + parents = list_delete_first(parents); + } while (parents); + + if (node->ps.plan->parallel_aware) + { + LWLockAcquire(&as_pstate->pa_lock, LW_EXCLUSIVE); + for (i = 0; i < node->as_nplans; i++) + as_pstate->pa_finished[i] = true; + LWLockRelease(&as_pstate->pa_lock); + } + + if (!bms_is_empty(subplans)) + { + while ((i = bms_first_member(subplans)) >= 0) + { + int index = plan->append_paths_array[i]; + + if (index >= 0) + { + node->subplan_indexes = lappend_int(node->subplan_indexes, + index); + + if (node->ps.plan->parallel_aware) + { + LWLockAcquire(&as_pstate->pa_lock, LW_EXCLUSIVE); + node->as_pstate->pa_finished[index] = false; + LWLockRelease(&as_pstate->pa_lock); + } + } + } + } + +} + +/* ---------------------------------------------------------------- * ExecInitAppend * * Begin all of the subscans of the append node. @@ -127,17 +211,19 @@ ExecInitAppend(Append *node, EState *estate, int eflags) appendstate->ps.ExecProcNode = ExecAppend; appendstate->appendplans = appendplanstates; appendstate->as_nplans = nplans; + appendstate->subplan_indexes = NIL; + appendstate->as_whichpartition = -1; /* * Miscellaneous initialization - * - * Append plans don't have expression contexts because they never call - * ExecQual or ExecProject. */ + /* create expression context for node */ + ExecAssignExprContext(estate, &appendstate->ps); + /* - * append nodes still have Result slots, which hold pointers to tuples, so - * we have to initialize them. + * append nodes have Result slots, which hold pointers to tuples, so we + * have to initialize them. */ ExecInitResultTupleSlot(estate, &appendstate->ps); @@ -154,6 +240,33 @@ ExecInitAppend(Append *node, EState *estate, int eflags) i++; } + node->join_exprstates = ExecInitExprList(node->join_clauses, + (PlanState *) appendstate); + + if (node->base_params || node->join_exprstates) + { + Relation rel; + PartitionDispatch *pd; + List *leaf_part_oids = NIL; + int num_parted; + + /* + * Get the information about the partition tree after locking all the + * partitions. + */ + rel = relation_open(node->parentoid, NoLock); + pd = RelationGetPartitionDispatchInfo(rel, &num_parted, &leaf_part_oids); + relation_close(rel, NoLock); + node->pd = pd; + for (i = 1; i < num_parted; i++) + { + PartitionDispatch partdispatch = pd[i]; + + heap_close(partdispatch->reldesc, NoLock); + ExecDropSingleTupleTableSlot(partdispatch->tupslot); + } + } + /* * initialize output tuple type */ @@ -184,12 +297,47 @@ static TupleTableSlot * ExecAppend(PlanState *pstate) { AppendState *node = castNode(AppendState, pstate); + Append *plan = (Append *) pstate->plan; + + /* Determine the subplans to scan depending on the base_params */ + if (plan->base_params && node->as_whichpartition == -1) + { + + if (node->ps.plan->parallel_aware) + { + ParallelAppendState *as_pstate = node->as_pstate; + LWLockAcquire(&as_pstate->prune_lock, LW_EXCLUSIVE); + + if(!as_pstate->is_runtime_pruned) + { + set_append_subplan_indexes(pstate, node, plan->base_params); + as_pstate->is_runtime_pruned = true; + if (node->subplan_indexes) + as_pstate->pa_next_plan = linitial_int(node->subplan_indexes);; + } + node->as_whichplan = INVALID_SUBPLAN_INDEX; + LWLockRelease(&as_pstate->prune_lock); + } + else + { + set_append_subplan_indexes(pstate, node, plan->base_params); + if (node->subplan_indexes) + { + node->as_whichplan = linitial_int(node->subplan_indexes); + node->as_whichpartition = 0; + } + } + } + +// if (plan->is_runtime_prunable && node->as_whichpartition == -1) +// return ExecClearTuple(node->ps.ps_ResultTupleSlot); /* If no subplan has been chosen, we must choose one before proceeding. */ if (node->as_whichplan == INVALID_SUBPLAN_INDEX && !node->choose_next_subplan(node)) return ExecClearTuple(node->ps.ps_ResultTupleSlot); + for (;;) { PlanState *subnode; @@ -256,6 +404,11 @@ void ExecReScanAppend(AppendState *node) { int i; + Append *plan = node->ps.plan; + + /* Determine subplans to scan based on the new Params */ + if (node->ps.chgParam != NULL && plan->join_exprstates) + set_append_subplan_indexes((PlanState *) node, node, plan->join_exprstates); for (i = 0; i < node->as_nplans; i++) { @@ -276,8 +429,21 @@ ExecReScanAppend(AppendState *node) ExecReScan(subnode); } - node->as_whichplan = - node->ps.plan->parallel_aware ? INVALID_SUBPLAN_INDEX : 0; + if (node->subplan_indexes) + { + if (node->ps.plan->parallel_aware) + node->as_pstate->pa_next_plan = linitial_int(node->subplan_indexes); + node->as_whichplan = + node->ps.plan->parallel_aware ? INVALID_SUBPLAN_INDEX : +linitial_int(node->subplan_indexes); + node->as_whichpartition = 0; + } + else + { + node->as_whichplan = + node->ps.plan->parallel_aware ? INVALID_SUBPLAN_INDEX : 0; + node->as_whichpartition = -1; + } } /* ---------------------------------------------------------------- @@ -320,6 +486,7 @@ ExecAppendInitializeDSM(AppendState *node, pstate = shm_toc_allocate(pcxt->toc, node->pstate_len); memset(pstate, 0, node->pstate_len); LWLockInitialize(&pstate->pa_lock, LWTRANCHE_PARALLEL_APPEND); + LWLockInitialize(&pstate->prune_lock, LWTRANCHE_PARALLEL_APPEND); shm_toc_insert(pcxt->toc, node->ps.plan->plan_node_id, pstate); node->as_pstate = pstate; @@ -353,6 +520,7 @@ ExecAppendInitializeWorker(AppendState *node, ParallelWorkerContext *pwcxt) { node->as_pstate = shm_toc_lookup(pwcxt->toc, node->ps.plan->plan_node_id, false); node->choose_next_subplan = choose_next_subplan_for_worker; +// sleep(30); } /* ---------------------------------------------------------------- @@ -365,22 +533,50 @@ ExecAppendInitializeWorker(AppendState *node, ParallelWorkerContext *pwcxt) static bool choose_next_subplan_locally(AppendState *node) { + Append *plan = (Append *) node->ps.plan; int whichplan = node->as_whichplan; /* We should never see INVALID_SUBPLAN_INDEX in this case. */ Assert(whichplan >= 0 && whichplan <= node->as_nplans); + /* Return false if no partition is selected by runtime pruning logic */ + if (plan->is_runtime_prunable && node->as_whichpartition == -1) + return false; + if (ScanDirectionIsForward(node->ps.state->es_direction)) { - if (whichplan >= node->as_nplans - 1) + /* + * For runtime partition pruning, goto the next valid partition index + */ + if (plan->is_runtime_prunable) + { + if (++node->as_whichpartition < list_length(node->subplan_indexes)) + node->as_whichplan = list_nth_int(node->subplan_indexes, + node->as_whichpartition); + else + return false; + } + + else if (whichplan >= node->as_nplans - 1) return false; - node->as_whichplan++; + else + node->as_whichplan++; } else { if (whichplan <= 0) return false; - node->as_whichplan--; + + if (node->subplan_indexes) + { + if (--(node->as_whichpartition) >= 0) + node->as_whichplan = list_nth_int(node->subplan_indexes, + node->as_whichpartition); + else + return false; + } + else + node->as_whichplan--; } return true; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index b1515dd..c46f439 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -243,6 +243,12 @@ _copyAppend(const Append *from) COPY_NODE_FIELD(partitioned_rels); COPY_NODE_FIELD(appendplans); COPY_SCALAR_FIELD(first_partial_plan); + COPY_SCALAR_FIELD(is_runtime_prunable); + COPY_SCALAR_FIELD(parentoid); + COPY_NODE_FIELD(base_params); + COPY_NODE_FIELD(join_clauses); + COPY_SCALAR_FIELD(append_paths_size); + COPY_POINTER_FIELD(append_paths_array, from->append_paths_size * sizeof(int)); return newnode; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index b59a521..0f4c907 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -393,6 +393,7 @@ _outModifyTable(StringInfo str, const ModifyTable *node) static void _outAppend(StringInfo str, const Append *node) { + int i; WRITE_NODE_TYPE("APPEND"); _outPlanInfo(str, (const Plan *) node); @@ -400,6 +401,13 @@ _outAppend(StringInfo str, const Append *node) WRITE_NODE_FIELD(partitioned_rels); WRITE_NODE_FIELD(appendplans); WRITE_INT_FIELD(first_partial_plan); + WRITE_NODE_FIELD(base_params); + WRITE_NODE_FIELD(join_clauses); + WRITE_OID_FIELD(parentoid); + WRITE_INT_FIELD(append_paths_size); + appendStringInfoString(str, " :append_paths_array"); + for (i = 0; i < node->append_paths_size; i++) + appendStringInfo(str, " %d", node->append_paths_array[i]); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 0d17ae8..8dd2fe3 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1601,7 +1601,11 @@ _readAppend(void) READ_NODE_FIELD(partitioned_rels); READ_NODE_FIELD(appendplans); READ_INT_FIELD(first_partial_plan); - + READ_NODE_FIELD(base_params); + READ_NODE_FIELD(join_clauses); + READ_OID_FIELD(parentoid); + READ_INT_FIELD(append_paths_size); + READ_INT_ARRAY(append_paths_array, local_node->append_paths_size); READ_DONE(); } diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index d222eff..609a7ca 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -25,6 +25,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" +#include "catalog/pg_inherits_fn.h" #include "foreign/fdwapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -144,9 +145,10 @@ static List *get_append_rel_partitions(PlannerInfo *root, static List *match_clauses_to_partkey(PlannerInfo *root, RelOptInfo *rel, List *clauses, + bool *contains_param, bool *contains_const, bool *constfalse); - +static int list_member_oid_index(List *list, Oid datum); /* * make_one_rel @@ -289,6 +291,27 @@ set_base_rel_sizes(PlannerInfo *root) if (root->glob->parallelModeOK) set_rel_consider_parallel(root, rel, rte); + if (rte->relkind == RELKIND_PARTITIONED_TABLE) + { + List *leaf_parts = NIL; + Relation parent = relation_open(rte->relid, NoLock); + int i; + + /* fetch the leaf oids of the parent rel */ + (void) find_all_inheritors(RelationGetRelid(parent), AccessShareLock, NULL); + get_leaf_part_recurse(parent, &leaf_parts); + relation_close(parent, NoLock); + + /* initialize necessary root variables */ + root->append_paths_size = list_length(leaf_parts); + root->leaf_node_oids = leaf_parts; + root->append_paths_array = palloc0(root->append_paths_size * sizeof(int)); + root->baserestrictinfo_param_indexes = NIL; + root->append_paths_count = 0; + + for (i = 0; i < root->append_paths_size; i++) + root->append_paths_array[i] = -1; + } set_rel_size(root, rel, rti, rte); } } @@ -350,6 +373,40 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, { /* It's an "append relation", process accordingly */ set_append_rel_size(root, rel, rti, rte); + + /* + * If this rel is part of a join then collect the necessary join + * clauses required for runtime partition pruning. + */ + if (rel->joininfo && rel->part_scheme) + { + List *partclauses; + bool contains_param, + contains_const, + constfalse; + + /* + * Get the clauses that match the partition key + */ + partclauses = match_clauses_to_partkey(root, rel, + list_copy(rel->joininfo), + &contains_param, + &contains_const, + &constfalse); + + if (partclauses != NIL) + { + ListCell *lc; + + foreach(lc, partclauses) + { + Node *n = lfirst(lc); + + if (!list_member(root->join_clauses, n)) + root->join_clauses = lappend(root->join_clauses, n); + } + } + } } else { @@ -871,7 +928,8 @@ get_append_rel_partitions(PlannerInfo *root, { List *partclauses; bool contains_const, - constfalse; + constfalse, + contains_param; List *result = NIL; int i; Relation parent; @@ -885,6 +943,7 @@ get_append_rel_partitions(PlannerInfo *root, */ partclauses = match_clauses_to_partkey(root, rel, list_copy(rel->baserestrictinfo), + &contains_param, &contains_const, &constfalse); @@ -901,7 +960,7 @@ get_append_rel_partitions(PlannerInfo *root, */ if (partclauses != NIL && contains_const) partindexes = get_partitions_from_clauses(parent, rel->relid, - partclauses); + partclauses, NULL, NULL); else { /* @@ -962,17 +1021,21 @@ get_append_rel_partitions(PlannerInfo *root, * * If the list contains a pseudo-constant RestrictInfo with constant false * value, *constfalse is set. + * + * If the list has a param, *contains_param is set */ static List * match_clauses_to_partkey(PlannerInfo *root, RelOptInfo *rel, List *clauses, + bool *contains_param, bool *contains_const, bool *constfalse) { PartitionScheme partscheme = rel->part_scheme; List *result = NIL; ListCell *lc; + int clause_index = -1; *contains_const = false; *constfalse = false; @@ -985,6 +1048,8 @@ match_clauses_to_partkey(PlannerInfo *root, Expr *clause; int i; + clause_index++; + if (IsA(member, RestrictInfo)) { RestrictInfo *rinfo = (RestrictInfo *) member; @@ -1022,14 +1087,24 @@ match_clauses_to_partkey(PlannerInfo *root, bool arg_matches_key = false, matched_arg_contains_const = false, all_args_constfalse = true; + List *base_param_indexes; + + /* + * Make a copy of root's baserestrictinfo_param_indexes before + * the recursive call + */ + base_param_indexes = + list_copy(root->baserestrictinfo_param_indexes); foreach (lc1, orclause->args) { Node *arg = lfirst(lc1); - bool contains_const1, - constfalse1; + bool contains_const1; + bool constfalse1; + bool contains_param1; if (match_clauses_to_partkey(root, rel, list_make1(arg), + &contains_param1, &contains_const1, &constfalse1) != NIL) { @@ -1040,8 +1115,22 @@ match_clauses_to_partkey(PlannerInfo *root, /* We got at least one arg that is not constant false. */ if (!constfalse1) all_args_constfalse = false; + + /* + * We got at least one arg which is Param, so add the + * current clause_index to the base_param_indexes list. + */ + if (contains_param1) + { + *contains_param = true; + base_param_indexes = + list_append_unique_int(base_param_indexes, + clause_index); + } } + root->baserestrictinfo_param_indexes = base_param_indexes; + if (arg_matches_key) { result = lappend(result, clause); @@ -1148,6 +1237,15 @@ match_clauses_to_partkey(PlannerInfo *root, if (contain_volatile_functions((Node *) constexpr)) continue; + if (IsA(constexpr, Param)) + { + *contains_param = true; + + root->baserestrictinfo_param_indexes = + list_append_unique_int(root->baserestrictinfo_param_indexes, + clause_index); + } + /* * Everything seems to be fine, so add it to the list of * clauses we will use for pruning. @@ -1247,6 +1345,26 @@ match_clauses_to_partkey(PlannerInfo *root, return result; } +/* list_member_oid_index + * Returns the index of the given datum in the oid list. + * + * It returns -1 if the datum is not found. + */ +static int +list_member_oid_index(List *list, Oid datum) +{ + int i = 0; + const ListCell *cell; + + foreach(cell, list) + { + if (lfirst_oid(cell) == datum) + return i; + i++; + } + return -1; +} + /* * set_append_rel_size * Set size estimates for a simple "append relation" @@ -1643,7 +1761,8 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, foreach(l, rel_appinfos) { AppendRelInfo *appinfo = (AppendRelInfo *) lfirst(l); - int childRTindex; + int childRTindex, + index; RangeTblEntry *childRTE; RelOptInfo *childrel; @@ -1676,6 +1795,11 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, if (IS_DUMMY_REL(childrel)) continue; + /* only consider non dummy children */ + index = list_member_oid_index(root->leaf_node_oids, childRTE->relid); + if (index >= 0) + root->append_paths_array[index] = root->append_paths_count++; + /* * Child is live, so add it to the live_childrels list for use below. */ @@ -1911,7 +2035,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, * if we have zero or one live subpath due to constraint exclusion.) */ if (subpaths_valid) - add_path(rel, (Path *) create_append_path(rel, subpaths, NIL, + add_path(rel, (Path *) create_append_path(root, rel, subpaths, NIL, NULL, 0, false, partitioned_rels, -1)); @@ -1953,7 +2077,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, Assert(parallel_workers > 0); /* Generate a partial append path. */ - appendpath = create_append_path(rel, NIL, partial_subpaths, NULL, + appendpath = create_append_path(root, rel, NIL, partial_subpaths, NULL, parallel_workers, enable_parallel_append, partitioned_rels, -1); @@ -2002,7 +2126,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, max_parallel_workers_per_gather); Assert(parallel_workers > 0); - appendpath = create_append_path(rel, pa_nonpartial_subpaths, + appendpath = create_append_path(root, rel, pa_nonpartial_subpaths, pa_partial_subpaths, NULL, parallel_workers, true, partitioned_rels, partial_rows); @@ -2058,7 +2182,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (subpaths_valid) add_path(rel, (Path *) - create_append_path(rel, subpaths, NIL, + create_append_path(root, rel, subpaths, NIL, required_outer, 0, false, partitioned_rels, -1)); } @@ -2323,7 +2447,7 @@ set_dummy_rel_pathlist(RelOptInfo *rel) rel->pathlist = NIL; rel->partial_pathlist = NIL; - add_path(rel, (Path *) create_append_path(rel, NIL, NIL, NULL, + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NIL, NULL, 0, false, NIL, -1)); /* diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 5bd3031..26a5f6e 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -1232,7 +1232,7 @@ mark_dummy_rel(RelOptInfo *rel) rel->partial_pathlist = NIL; /* Set up the dummy path */ - add_path(rel, (Path *) create_append_path(rel, NIL, NIL, NULL, + add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NIL, NULL, 0, false, NIL, -1)); /* Set or update cheapest_total_path and related fields */ diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index f6c83d0..936a0b8 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1002,6 +1002,24 @@ create_join_plan(PlannerInfo *root, JoinPath *best_path) return plan; } +static List * +replace_partition_nestloop_params(PlannerInfo *root, List *old_list) +{ + List *new_list = NIL; + ListCell *lc; + + if (old_list == NULL) + return NULL; + + foreach(lc, old_list) + { + Node *n = lfirst(lc); + + new_list = lappend(new_list, replace_nestloop_params(root, n)); + } + return new_list; +} + /* * create_append_plan * Create an Append plan for 'best_path' and (recursively) plans @@ -1065,6 +1083,14 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path) copy_generic_path_info(&plan->plan, (Path *) best_path); + plan->base_params = best_path->base_params; + plan->join_clauses = replace_partition_nestloop_params(root, best_path->join_clauses); + plan->parentoid = best_path->parentoid; + plan->parentrelid = best_path->path.parent->relid; + plan->append_paths_size = best_path->append_paths_size; + plan->append_paths_array = best_path->append_paths_array; + plan->is_runtime_prunable = (plan->join_clauses || plan->base_params); + return (Plan *) plan; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 7a09f07..565d5b7 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3678,7 +3678,7 @@ create_grouping_paths(PlannerInfo *root, paths = lappend(paths, path); } path = (Path *) - create_append_path(grouped_rel, + create_append_path(root, grouped_rel, paths, NIL, NULL, diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index a24e8ac..aab53dd 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -590,7 +590,7 @@ generate_union_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NIL, + path = (Path *) create_append_path(root, result_rel, pathlist, NIL, NULL, 0, false, NIL, -1); /* We have to manually jam the right tlist into the path; ick */ path->pathtarget = create_pathtarget(root, tlist); @@ -702,7 +702,7 @@ generate_nonunion_path(SetOperationStmt *op, PlannerInfo *root, /* * Append the child results together. */ - path = (Path *) create_append_path(result_rel, pathlist, NIL, + path = (Path *) create_append_path(root, result_rel, pathlist, NIL, NULL, 0, false, NIL, -1); /* We have to manually jam the right tlist into the path; ick */ diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index ce83fbc..60f31e3 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -2470,6 +2470,25 @@ eval_const_expressions(PlannerInfo *root, Node *node) } /*-------------------- + * eval_const_expressions_from_list + * + * This is similar to eval_const_expression except that it takes ParamListInfo + * argument instead of PlannerInfo to create the context. + */ +Node * +eval_const_expressions_from_list(ParamListInfo prmlist, Node *node) +{ + eval_const_expressions_context context; + + context.boundParams = prmlist; /* bound Params */ + context.root = NULL; + context.active_fns = NIL; /* nothing being recursively simplified */ + context.case_val = NULL; /* no CASE being examined */ + context.estimate = false; /* safe transformations only */ + return eval_const_expressions_mutator(node, &context); +} + +/*-------------------- * estimate_expression_value * * This function attempts to estimate the value of an expression for diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 54126fb..7091a4f 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1210,7 +1210,8 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, * Note that we must handle subpaths = NIL, representing a dummy access path. */ AppendPath * -create_append_path(RelOptInfo *rel, +create_append_path(PlannerInfo *root, + RelOptInfo *rel, List *subpaths, List *partial_subpaths, Relids required_outer, int parallel_workers, bool parallel_aware, @@ -1269,6 +1270,56 @@ create_append_path(RelOptInfo *rel, if (rows >= 0) pathnode->path.rows = rows; + /* Do necessary evaluation needed for runtime partition pruning. */ + if (root && (required_outer || rel->baserestrictinfo)) + { + RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); + + pathnode->append_paths_array = root->append_paths_array; + pathnode->append_paths_size = root->append_paths_size; + + if (rte && rte->rtekind == RTE_RELATION) + { + Oid poid = rte->relid; + Relation prel = relation_open(poid, NoLock); + + if (prel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + if (root->join_clauses) + pathnode->join_clauses = root->join_clauses; + + if (root->baserestrictinfo_param_indexes && rel->baserestrictinfo) + { + ListCell *lc; + List *quals = list_copy(rel->baserestrictinfo); + + pathnode->base_params = NIL; + + /* + * collect the clauses from baserestrictinfo for partition + * pruning + */ + foreach(lc, root->baserestrictinfo_param_indexes) + { + int index = lfirst_int(lc); + Node *member = list_nth(quals, index); + Expr *clause; + + if (IsA(member, RestrictInfo)) + clause = ((RestrictInfo *) member)->clause; + else + clause = (Expr *) member; + + pathnode->base_params = lappend(pathnode->base_params, + clause); + } + } + pathnode->parentoid = poid; + } + relation_close(prel, NoLock); + } + } + return pathnode; } diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index dcfda1c..fc7cbcd 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -1554,6 +1554,9 @@ ParamPathInfo * get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) { ParamPathInfo *ppi; + Relids joinrelids; + List *pclauses; + ListCell *lc; /* Unparameterized paths have no ParamPathInfo */ if (bms_is_empty(required_outer)) @@ -1565,11 +1568,27 @@ get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) if ((ppi = find_param_path_info(appendrel, required_outer))) return ppi; + /* + * To determine whether the appendrel is applicable for runtime pruning or + * not, we fetch the clause from the join clause. + */ + joinrelids = bms_union(appendrel->relids, required_outer); + pclauses = NIL; + foreach(lc, appendrel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + if (join_clause_is_movable_into(rinfo, + appendrel->relids, + joinrelids)) + pclauses = lappend(pclauses, rinfo); + } + /* Else build the ParamPathInfo */ ppi = makeNode(ParamPathInfo); ppi->ppi_req_outer = required_outer; ppi->ppi_rows = 0; - ppi->ppi_clauses = NIL; + ppi->ppi_clauses = pclauses; appendrel->ppilist = lappend(appendrel->ppilist, ppi); return ppi; diff --git a/src/include/catalog/partition.h b/src/include/catalog/partition.h index 7a5ab45..c4a7788 100644 --- a/src/include/catalog/partition.h +++ b/src/include/catalog/partition.h @@ -16,6 +16,7 @@ #include "fmgr.h" #include "executor/tuptable.h" #include "nodes/execnodes.h" +#include "nodes/relation.h" #include "parser/parse_node.h" #include "utils/rel.h" @@ -73,5 +74,7 @@ extern int get_partition_for_tuple(Relation relation, Datum *values, /* For partition-pruning */ extern Bitmapset *get_partitions_from_clauses(Relation relation, int rt_index, - List *partclauses); + List *partclauses, ParamListInfo base_prmlist, + ExprContext *econtext); +extern void get_leaf_part_recurse(Relation rel, List **leaf_part_oids); #endif /* PARTITION_H */ diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 86a199d..e6fea65 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -49,6 +49,9 @@ typedef struct PartitionDispatchData typedef struct PartitionDispatchData *PartitionDispatch; +extern PartitionDispatch *RelationGetPartitionDispatchInfo(Relation rel, + int *num_parted, List **leaf_part_oids); + extern void ExecSetupPartitionTupleRouting(ModifyTableState *mtstate, Relation rel, Index resultRTindex, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 1a35c5c..f0ac6bc 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1006,6 +1006,7 @@ struct AppendState; typedef struct AppendState AppendState; struct ParallelAppendState; typedef struct ParallelAppendState ParallelAppendState; +struct PartitionDispatchData; struct AppendState { @@ -1016,6 +1017,8 @@ struct AppendState ParallelAppendState *as_pstate; /* parallel coordination info */ Size pstate_len; /* size of parallel coordination info */ bool (*choose_next_subplan) (AppendState *); + int as_whichpartition; + List *subplan_indexes; }; /* ---------------- diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 02fb366..f04105e 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -19,6 +19,7 @@ #include "nodes/bitmapset.h" #include "nodes/lockoptions.h" #include "nodes/primnodes.h" +#include "nodes/relation.h" /* ---------------------------------------------------------------- @@ -242,6 +243,7 @@ typedef struct ModifyTable * Generate the concatenation of the results of sub-plans. * ---------------- */ +struct PartitionDispatchData; typedef struct Append { Plan plan; @@ -249,6 +251,17 @@ typedef struct Append List *partitioned_rels; List *appendplans; int first_partial_plan; + + /* for runtime partition pruning */ + bool is_runtime_prunable; + Oid parentoid; + Index parentrelid; + List *base_params; + List *join_clauses; + List *join_exprstates; + int *append_paths_array; + int append_paths_size; + struct PartitionDispatchData **pd; } Append; /* ---------------- diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index bcb669d..b7b6945 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -317,6 +317,17 @@ typedef struct PlannerInfo /* optional private data for join_search_hook, e.g., GEQO */ void *join_search_private; + + /* These fields accumulate data required for runtime partition pruning. */ + int *append_paths_array; /* array for append subplans info */ + int append_paths_size; /* size of append_paths_array */ + int append_paths_count; /* current count of append_paths_array */ + List *leaf_node_oids; /* leaf oids of current rel */ + List *baserestrictinfo_param_indexes; /* index from baserestrictinfo + * list with partition related + * clauses */ + List *join_clauses; /* join clause which are required for + * partition pruning */ } PlannerInfo; @@ -1295,6 +1306,13 @@ typedef struct AppendPath /* Index of first partial path in subpaths */ int first_partial_path; + + /* for runtime partition pruning */ + Oid parentoid; + int *append_paths_array; + int append_paths_size; + List *base_params; + List *join_clauses; } AppendPath; #define IS_DUMMY_PATH(p) \ diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index 1ef13a4..f8e7660 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -79,6 +79,7 @@ extern void CommuteOpExpr(OpExpr *clause); extern void CommuteRowCompareExpr(RowCompareExpr *clause); extern Node *eval_const_expressions(PlannerInfo *root, Node *node); +extern Node *eval_const_expressions_from_list(ParamListInfo prmlist, Node *node); extern Node *estimate_expression_value(PlannerInfo *root, Node *node); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 00c134d..2ac3463 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -64,9 +64,9 @@ extern BitmapOrPath *create_bitmap_or_path(PlannerInfo *root, List *bitmapquals); extern TidPath *create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, Relids required_outer); -extern AppendPath *create_append_path(RelOptInfo *rel, - List *subpaths, List *partial_subpaths, - Relids required_outer, +extern AppendPath *create_append_path(PlannerInfo *root, + RelOptInfo *rel, List *subpaths, + List *partial_subpaths, Relids required_outer, int parallel_workers, bool parallel_aware, List *partitioned_rels, double rows); extern MergeAppendPath *create_merge_append_path(PlannerInfo *root,