From 7a3e86a079be3fb6e6c8fa4a2ed8f0101fe0ee5b Mon Sep 17 00:00:00 2001 From: Amit Langote Date: Wed, 7 Aug 2024 18:25:51 +0900 Subject: [PATCH v53 3/4] Defer locking of runtime-prunable relations to executor When preparing a cached plan for execution, plancache.c locks the relations contained in the plan's range table to ensure it is safe for execution. However, this simplistic approach, implemented in AcquireExecutorLocks(), results in unnecessarily locking relations that might be pruned during "initial" runtime pruning. To optimize this, the locking is now deferred for relations that are subject to "initial" runtime pruning. The planner now provides a set of "unprunable" relations, available through the new PlannedStmt.unprunableRelids field. AcquireExecutorLocks() will now only lock those relations. PlannedStmt.unprunableRelids is populated by subtracting the set of initially prunable relids from the set of all RT indexes. The prunable relids set is constructed by examining all PartitionPruneInfos during set_plan_refs() and storing the RT indexes of partitions subject to "initial" pruning steps. While at it, some duplicated code in set_append_references() and set_mergeappend_references() that constructs the prunable relids set has been refactored into a common function. To enable the executor to determine whether the plan tree it's executing is a cached one, the CachedPlan is now made available via the QueryDesc. The executor can call CachedPlanRequiresLocking(), which returns true if the CachedPlan is a reusable generic plan that might contain relations needing to be locked. If so, the executor will lock any relation that is not in PlannedStmt.unprunableRelids. Finally, an Assert has been added in ExecCheckPermissions() to ensure that all relations whose permissions are checked have been properly locked. This helps catch any accidental omission of relations from the unprunableRelids set that should have their permissions checked. This deferment introduces a window in which prunable relations may be altered by concurrent DDL, potentially causing the plan to become invalid. As a result, the executor might attempt to run an invalid plan, leading to errors such as being unable to locate a partition-only index during ExecInitIndexScan(). Future commits will introduce changes to ready the executor to check plan validity during ExecutorStart() and retry with a newly created plan if the original one becomes invalid after taking deferred locks. --- src/backend/commands/copyto.c | 2 +- src/backend/commands/createas.c | 2 +- src/backend/commands/explain.c | 7 ++-- src/backend/commands/extension.c | 1 + src/backend/commands/matview.c | 2 +- src/backend/commands/prepare.c | 3 +- src/backend/executor/execMain.c | 45 +++++++++++++++++++++++++- src/backend/executor/execParallel.c | 9 +++++- src/backend/executor/execPartition.c | 37 ++++++++++++++++++--- src/backend/executor/functions.c | 1 + src/backend/executor/nodeAppend.c | 8 ++--- src/backend/executor/nodeMergeAppend.c | 2 +- src/backend/executor/spi.c | 1 + src/backend/optimizer/plan/planner.c | 2 ++ src/backend/optimizer/plan/setrefs.c | 11 +++++++ src/backend/partitioning/partprune.c | 20 +++++++++++- src/backend/tcop/pquery.c | 10 +++++- src/backend/utils/cache/plancache.c | 40 ++++++++++++++--------- src/include/commands/explain.h | 5 +-- src/include/executor/execPartition.h | 9 +++++- src/include/executor/execdesc.h | 2 ++ src/include/nodes/execnodes.h | 2 ++ src/include/nodes/pathnodes.h | 6 ++++ src/include/nodes/plannodes.h | 11 +++++++ src/include/utils/plancache.h | 10 ++++++ 25 files changed, 209 insertions(+), 39 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 91de442f43..db976f928a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -552,7 +552,7 @@ BeginCopyTo(ParseState *pstate, ((DR_copy *) dest)->cstate = cstate; /* Create a QueryDesc requesting no output */ - cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, + cstate->queryDesc = CreateQueryDesc(plan, NULL, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 0b629b1f79..57a3375cad 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -324,7 +324,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, UpdateActiveSnapshotCommandId(); /* Create a QueryDesc, redirecting output to our tuple receiver */ - queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, + queryDesc = CreateQueryDesc(plan, NULL, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, 0); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 2819e479f8..13f5683cf6 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -507,7 +507,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, } /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + ExplainOnePlan(plan, NULL, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), es->memory ? &mem_counters : NULL); } @@ -615,7 +615,8 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, * to call it. */ void -ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, +ExplainOnePlan(PlannedStmt *plannedstmt, CachedPlan *cplan, + IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, @@ -671,7 +672,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, dest = None_Receiver; /* Create a QueryDesc for the query */ - queryDesc = CreateQueryDesc(plannedstmt, queryString, + queryDesc = CreateQueryDesc(plannedstmt, cplan, queryString, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, instrument_option); diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index fab59ad5f6..bd169edeff 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -742,6 +742,7 @@ execute_sql_string(const char *sql) QueryDesc *qdesc; qdesc = CreateQueryDesc(stmt, + NULL, sql, GetActiveSnapshot(), NULL, dest, NULL, NULL, 0); diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 010097873d..69be74b4bd 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -438,7 +438,7 @@ refresh_matview_datafill(DestReceiver *dest, Query *query, UpdateActiveSnapshotCommandId(); /* Create a QueryDesc, redirecting output to our tuple receiver */ - queryDesc = CreateQueryDesc(plan, queryString, + queryDesc = CreateQueryDesc(plan, NULL, queryString, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index 07257d4db9..311b9ebd5b 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -655,7 +655,8 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, PlannedStmt *pstmt = lfirst_node(PlannedStmt, p); if (pstmt->commandType != CMD_UTILITY) - ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv, + ExplainOnePlan(pstmt, cplan, into, es, query_string, paramLI, + queryEnv, &planduration, (es->buffers ? &bufusage : NULL), es->memory ? &mem_counters : NULL); else diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index dceef322af..cb5ed921d0 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -53,6 +53,7 @@ #include "miscadmin.h" #include "parser/parse_relation.h" #include "rewrite/rewriteHandler.h" +#include "storage/lmgr.h" #include "tcop/utility.h" #include "utils/acl.h" #include "utils/backend_status.h" @@ -90,6 +91,7 @@ static bool ExecCheckPermissionsModified(Oid relOid, Oid userid, AclMode requiredPerms); static void ExecCheckXactReadOnly(PlannedStmt *plannedstmt); static void EvalPlanQualStart(EPQState *epqstate, Plan *planTree); +static inline bool ExecShouldLockRelations(EState *estate); /* end of local decls */ @@ -598,6 +600,21 @@ ExecCheckPermissions(List *rangeTable, List *rteperminfos, (rte->rtekind == RTE_SUBQUERY && rte->relkind == RELKIND_VIEW)); + /* + * Ensure that we have at least an AccessShareLock on relations + * whose permissions need to be checked. + * + * Skip this check in a parallel worker because locks won't be + * taken until ExecInitNode() performs plan initialization. + * + * XXX: ExecCheckPermissions() in a parallel worker may be + * redundant with the checks done in the leader process, so this + * should be reviewed to ensure it’s necessary. + */ + Assert(IsParallelWorker() || + CheckRelationOidLockedByMe(rte->relid, AccessShareLock, + true)); + (void) getRTEPermissionInfo(rteperminfos, rte); /* Many-to-one mapping not allowed */ Assert(!bms_is_member(rte->perminfoindex, indexset)); @@ -860,11 +877,35 @@ ExecDoInitialPruning(EState *estate) * result. */ if (prunestate->do_initial_prune) - validsubplans = ExecFindMatchingSubPlans(prunestate, true); + { + List *leaf_partition_oids = NIL; + + validsubplans = ExecFindMatchingSubPlans(prunestate, true, + &leaf_partition_oids); + if (ExecShouldLockRelations(estate)) + { + ListCell *lc1; + + foreach(lc1, leaf_partition_oids) + { + LockRelationOid(lfirst_oid(lc1), prunestate->lockmode); + } + } + } estate->es_part_prune_results = lappend(estate->es_part_prune_results, validsubplans); } } +/* + * Locks might be needed only if running a cached plan that might contain + * unlocked relations, such as reused generic plans. + */ +static inline bool +ExecShouldLockRelations(EState *estate) +{ + return estate->es_cachedplan == NULL ? false : + CachedPlanRequiresLocking(estate->es_cachedplan); +} /* ---------------------------------------------------------------- * InitPlan @@ -878,6 +919,7 @@ InitPlan(QueryDesc *queryDesc, int eflags) { CmdType operation = queryDesc->operation; PlannedStmt *plannedstmt = queryDesc->plannedstmt; + CachedPlan *cachedplan = queryDesc->cplan; Plan *plan = plannedstmt->planTree; List *rangeTable = plannedstmt->rtable; EState *estate = queryDesc->estate; @@ -897,6 +939,7 @@ InitPlan(QueryDesc *queryDesc, int eflags) ExecInitRangeTable(estate, rangeTable, plannedstmt->permInfos); estate->es_plannedstmt = plannedstmt; + estate->es_cachedplan = cachedplan; /* * Perform runtime "initial" pruning to determine the plan nodes that will diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index b01a2fdfdd..7519c9a860 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -1257,8 +1257,15 @@ ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver, paramspace = shm_toc_lookup(toc, PARALLEL_KEY_PARAMLISTINFO, false); paramLI = RestoreParamList(¶mspace); - /* Create a QueryDesc for the query. */ + /* + * Create a QueryDesc for the query. We pass NULL for cachedplan, because + * we don't have a pointer to the CachedPlan in the leader's process. It's + * fine because the only reason the executor needs to see it is to decide + * if it should take locks on certain relations, but paraller workers + * always take locks anyway. + */ return CreateQueryDesc(pstmt, + NULL, queryString, GetActiveSnapshot(), InvalidSnapshot, receiver, paramLI, NULL, instrument_options); diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 08b1f3d030..861e64856d 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -26,6 +26,7 @@ #include "partitioning/partdesc.h" #include "partitioning/partprune.h" #include "rewrite/rewriteManip.h" +#include "storage/lmgr.h" #include "utils/acl.h" #include "utils/lsyscache.h" #include "utils/partcache.h" @@ -196,7 +197,8 @@ static void PartitionPruneInitExecPruning(PartitionPruneInfo *pruneinfo, static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, - Bitmapset **validsubplans); + Bitmapset **validsubplans, + List **leaf_part_oids); /* @@ -1927,6 +1929,7 @@ CreatePartitionPruneState(EState *estate, PartitionPruneInfo *pruneinfo) ALLOCSET_DEFAULT_SIZES); i = 0; + prunestate->lockmode = NoLock; foreach(lc, pruneinfo->prune_infos) { List *partrelpruneinfos = lfirst_node(List, lc); @@ -1950,6 +1953,15 @@ CreatePartitionPruneState(EState *estate, PartitionPruneInfo *pruneinfo) PartitionDesc partdesc; PartitionKey partkey; + /* + * Assign the lock mode of the first (root) partitioned table's RTE + * as the lock mode to lock leaf partitions after initial pruning, + * if needed. + */ + if (prunestate->lockmode == NoLock) + prunestate->lockmode = exec_rt_fetch(pinfo->rtindex, estate)->rellockmode; + Assert(prunestate->lockmode != NoLock); + /* * We can rely on the copies of the partitioned table's partition * key and partition descriptor appearing in its relcache entry, @@ -1982,6 +1994,9 @@ CreatePartitionPruneState(EState *estate, PartitionPruneInfo *pruneinfo) pprune->partrel = partrel; pprune->nparts = partdesc->nparts; pprune->subplan_map = palloc(sizeof(int) * partdesc->nparts); + pprune->relid_map = palloc(sizeof(Oid) * partdesc->nparts); + memcpy(pprune->relid_map, partdesc->oids, + sizeof(Oid) * partdesc->nparts); if (partdesc->nparts == pinfo->nparts && memcmp(partdesc->oids, pinfo->relid_map, @@ -2399,10 +2414,13 @@ PartitionPruneInitExecPruning(PartitionPruneInfo *pruneinfo, * Pass initial_prune if PARAM_EXEC Params cannot yet be evaluated. This * differentiates the initial executor-time pruning step from later * runtime pruning. + * + * leaf_part_oids must be non-NULL if initial_prune is true. */ Bitmapset * ExecFindMatchingSubPlans(PartitionPruneState *prunestate, - bool initial_prune) + bool initial_prune, + List **leaf_part_oids) { Bitmapset *result = NULL; MemoryContext oldcontext; @@ -2437,7 +2455,7 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate, */ pprune = &prunedata->partrelprunedata[0]; find_matching_subplans_recurse(prunedata, pprune, initial_prune, - &result); + &result, leaf_part_oids); /* Expression eval may have used space in ExprContext too */ if (pprune->exec_pruning_steps) @@ -2451,6 +2469,8 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate, /* Copy result out of the temp context before we reset it */ result = bms_copy(result); + if (leaf_part_oids) + *leaf_part_oids = list_copy(*leaf_part_oids); MemoryContextReset(prunestate->prune_context); @@ -2467,7 +2487,8 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, - Bitmapset **validsubplans) + Bitmapset **validsubplans, + List **leaf_part_oids) { Bitmapset *partset; int i; @@ -2494,8 +2515,13 @@ find_matching_subplans_recurse(PartitionPruningData *prunedata, while ((i = bms_next_member(partset, i)) >= 0) { if (pprune->subplan_map[i] >= 0) + { *validsubplans = bms_add_member(*validsubplans, pprune->subplan_map[i]); + if (leaf_part_oids) + *leaf_part_oids = lappend_oid(*leaf_part_oids, + pprune->relid_map[i]); + } else { int partidx = pprune->subpart_map[i]; @@ -2503,7 +2529,8 @@ find_matching_subplans_recurse(PartitionPruningData *prunedata, if (partidx >= 0) find_matching_subplans_recurse(prunedata, &prunedata->partrelprunedata[partidx], - initial_prune, validsubplans); + initial_prune, validsubplans, + leaf_part_oids); else { /* diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index 692854e2b3..6f6f45e0ad 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -840,6 +840,7 @@ postquel_start(execution_state *es, SQLFunctionCachePtr fcache) dest = None_Receiver; es->qd = CreateQueryDesc(es->stmt, + NULL, fcache->src, GetActiveSnapshot(), InvalidSnapshot, diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index de7ebab5c2..006bdafaea 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -581,7 +581,7 @@ choose_next_subplan_locally(AppendState *node) else if (!node->as_valid_subplans_identified) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); node->as_valid_subplans_identified = true; } @@ -648,7 +648,7 @@ choose_next_subplan_for_leader(AppendState *node) if (!node->as_valid_subplans_identified) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); node->as_valid_subplans_identified = true; /* @@ -724,7 +724,7 @@ choose_next_subplan_for_worker(AppendState *node) else if (!node->as_valid_subplans_identified) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); node->as_valid_subplans_identified = true; mark_invalid_subplans_as_finished(node); @@ -877,7 +877,7 @@ ExecAppendAsyncBegin(AppendState *node) if (!node->as_valid_subplans_identified) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); node->as_valid_subplans_identified = true; classify_matching_subplans(node); diff --git a/src/backend/executor/nodeMergeAppend.c b/src/backend/executor/nodeMergeAppend.c index 3ed91808dd..f7821aa178 100644 --- a/src/backend/executor/nodeMergeAppend.c +++ b/src/backend/executor/nodeMergeAppend.c @@ -219,7 +219,7 @@ ExecMergeAppend(PlanState *pstate) */ if (node->ms_valid_subplans == NULL) node->ms_valid_subplans = - ExecFindMatchingSubPlans(node->ms_prune_state, false); + ExecFindMatchingSubPlans(node->ms_prune_state, false, NULL); /* * First time through: pull the first tuple from each valid subplan, diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 90d9834576..659bd6dcd9 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -2684,6 +2684,7 @@ _SPI_execute_plan(SPIPlanPtr plan, const SPIExecuteOptions *options, snap = InvalidSnapshot; qdesc = CreateQueryDesc(stmt, + cplan, plansource->query_string, snap, crosscheck_snapshot, dest, diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 1b9071c774..9e47a7fd50 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -549,6 +549,8 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, result->planTree = top_plan; result->partPruneInfos = glob->partPruneInfos; result->rtable = glob->finalrtable; + result->unprunableRelids = bms_difference(bms_add_range(NULL, 1, list_length(result->rtable)), + glob->prunableRelids); result->permInfos = glob->finalrteperminfos; result->resultRelations = glob->resultRelations; result->appendRelations = glob->appendRelations; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index e2ea406c4e..8ce6d1149d 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -1764,8 +1764,19 @@ register_partpruneinfo(PlannerInfo *root, int part_prune_index, int rtoffset) foreach(l2, prune_infos) { PartitionedRelPruneInfo *prelinfo = lfirst(l2); + Bitmapset *present_leafpart_rtis = prelinfo->present_leafpart_rtis; prelinfo->rtindex += rtoffset; + present_leafpart_rtis = offset_relid_set(present_leafpart_rtis, + rtoffset); + if (prelinfo->initial_pruning_steps != NIL) + glob->prunableRelids = bms_add_members(glob->prunableRelids, + present_leafpart_rtis); + /* + * Don't need this anymore, so set to NULL to save space in the + * final plan tree. + */ + prelinfo->present_leafpart_rtis = NULL; } } diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 60fabb1734..c022c5ee0b 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -641,6 +641,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, PartitionedRelPruneInfo *pinfo = lfirst(lc); RelOptInfo *subpart = find_base_rel(root, pinfo->rtindex); Bitmapset *present_parts; + Bitmapset *present_leafpart_rtis; int nparts = subpart->nparts; int *subplan_map; int *subpart_map; @@ -657,7 +658,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, subpart_map = (int *) palloc(nparts * sizeof(int)); memset(subpart_map, -1, nparts * sizeof(int)); relid_map = (Oid *) palloc0(nparts * sizeof(Oid)); - present_parts = NULL; + present_parts = present_leafpart_rtis = NULL; i = -1; while ((i = bms_next_member(subpart->live_parts, i)) >= 0) @@ -671,9 +672,25 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, subplan_map[i] = subplanidx = relid_subplan_map[partrel->relid] - 1; subpart_map[i] = subpartidx = relid_subpart_map[partrel->relid] - 1; relid_map[i] = planner_rt_fetch(partrel->relid, root)->relid; + + /* + * Track the RT indexes of partitions to ensure they are included + * in the prunableRelids set of relations that are locked during + * execution. This ensures that if the plan is cached, these + * partitions are locked when the plan is reused. + * + * Partitions without a subplan and sub-partitioned partitions + * where none of the sub-partitions have a subplan due to + * constraint exclusion are not included in this set. Instead, + * they are added to the unprunableRelids set, and the relations + * in this set are locked by AcquireExecutorLocks() before + * executing a cached plan. + */ if (subplanidx >= 0) { present_parts = bms_add_member(present_parts, i); + present_leafpart_rtis = bms_add_member(present_leafpart_rtis, + partrel->relid); /* Record finding this subplan */ subplansfound = bms_add_member(subplansfound, subplanidx); @@ -691,6 +708,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, /* Record the maps and other information. */ pinfo->present_parts = present_parts; + pinfo->present_leafpart_rtis = present_leafpart_rtis; pinfo->nparts = nparts; pinfo->subplan_map = subplan_map; pinfo->subpart_map = subpart_map; diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index a1f8d03db1..6e8f6b1b8f 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -36,6 +36,7 @@ Portal ActivePortal = NULL; static void ProcessQuery(PlannedStmt *plan, + CachedPlan *cplan, const char *sourceText, ParamListInfo params, QueryEnvironment *queryEnv, @@ -65,6 +66,7 @@ static void DoPortalRewind(Portal portal); */ QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, + CachedPlan *cplan, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, @@ -77,6 +79,7 @@ CreateQueryDesc(PlannedStmt *plannedstmt, qd->operation = plannedstmt->commandType; /* operation */ qd->plannedstmt = plannedstmt; /* plan */ + qd->cplan = cplan; /* CachedPlan supplying the plannedstmt */ qd->sourceText = sourceText; /* query text */ qd->snapshot = RegisterSnapshot(snapshot); /* snapshot */ /* RI check snapshot */ @@ -122,6 +125,7 @@ FreeQueryDesc(QueryDesc *qdesc) * PORTAL_ONE_RETURNING, or PORTAL_ONE_MOD_WITH portal * * plan: the plan tree for the query + * cplan: CachedPlan supplying the plan * sourceText: the source text of the query * params: any parameters needed * dest: where to send results @@ -134,6 +138,7 @@ FreeQueryDesc(QueryDesc *qdesc) */ static void ProcessQuery(PlannedStmt *plan, + CachedPlan *cplan, const char *sourceText, ParamListInfo params, QueryEnvironment *queryEnv, @@ -145,7 +150,7 @@ ProcessQuery(PlannedStmt *plan, /* * Create the QueryDesc object */ - queryDesc = CreateQueryDesc(plan, sourceText, + queryDesc = CreateQueryDesc(plan, cplan, sourceText, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, 0); @@ -493,6 +498,7 @@ PortalStart(Portal portal, ParamListInfo params, * the destination to DestNone. */ queryDesc = CreateQueryDesc(linitial_node(PlannedStmt, portal->stmts), + portal->cplan, portal->sourceText, GetActiveSnapshot(), InvalidSnapshot, @@ -1276,6 +1282,7 @@ PortalRunMulti(Portal portal, { /* statement can set tag string */ ProcessQuery(pstmt, + portal->cplan, portal->sourceText, portal->portalParams, portal->queryEnv, @@ -1285,6 +1292,7 @@ PortalRunMulti(Portal portal, { /* stmt added by rewrite cannot set tag */ ProcessQuery(pstmt, + portal->cplan, portal->sourceText, portal->portalParams, portal->queryEnv, diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 5af1a168ec..5b75dadf13 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -104,7 +104,8 @@ static List *RevalidateCachedQuery(CachedPlanSource *plansource, QueryEnvironment *queryEnv); static bool CheckCachedPlan(CachedPlanSource *plansource); static CachedPlan *BuildCachedPlan(CachedPlanSource *plansource, List *qlist, - ParamListInfo boundParams, QueryEnvironment *queryEnv); + ParamListInfo boundParams, QueryEnvironment *queryEnv, + bool generic); static bool choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams); static double cached_plan_cost(CachedPlan *plan, bool include_planner); @@ -815,8 +816,11 @@ RevalidateCachedQuery(CachedPlanSource *plansource, * Caller must have already called RevalidateCachedQuery to verify that the * querytree is up to date. * - * On a "true" return, we have acquired the locks needed to run the plan. - * (We must do this for the "true" result to be race-condition-free.) + * On a "true" return, we have acquired locks on the "unprunableRelids" set + * for all plans in plansource->stmt_list. The plans are not completely + * race-condition-free until the executor takes locks on the set of prunable + * relations that survive initial runtime pruning during executor + * initialization; */ static bool CheckCachedPlan(CachedPlanSource *plansource) @@ -893,10 +897,10 @@ CheckCachedPlan(CachedPlanSource *plansource) * or it can be set to NIL if we need to re-copy the plansource's query_list. * * To build a generic, parameter-value-independent plan, pass NULL for - * boundParams. To build a custom plan, pass the actual parameter values via - * boundParams. For best effect, the PARAM_FLAG_CONST flag should be set on - * each parameter value; otherwise the planner will treat the value as a - * hint rather than a hard constant. + * boundParams, and true for generic. To build a custom plan, pass the actual + * parameter values via boundParams, and false for generic. For best effect, + * the PARAM_FLAG_CONST flag should be set on each parameter value; otherwise + * the planner will treat the value as a hint rather than a hard constant. * * Planning work is done in the caller's memory context. The finished plan * is in a child memory context, which typically should get reparented @@ -904,7 +908,8 @@ CheckCachedPlan(CachedPlanSource *plansource) */ static CachedPlan * BuildCachedPlan(CachedPlanSource *plansource, List *qlist, - ParamListInfo boundParams, QueryEnvironment *queryEnv) + ParamListInfo boundParams, QueryEnvironment *queryEnv, + bool generic) { CachedPlan *plan; List *plist; @@ -1026,6 +1031,7 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, plan->refcount = 0; plan->context = plan_context; plan->is_oneshot = plansource->is_oneshot; + plan->is_generic = generic; plan->is_saved = false; plan->is_valid = true; @@ -1196,7 +1202,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, else { /* Build a new generic plan */ - plan = BuildCachedPlan(plansource, qlist, NULL, queryEnv); + plan = BuildCachedPlan(plansource, qlist, NULL, queryEnv, true); /* Just make real sure plansource->gplan is clear */ ReleaseGenericPlan(plansource); /* Link the new generic plan into the plansource */ @@ -1241,7 +1247,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, if (customplan) { /* Build a custom plan */ - plan = BuildCachedPlan(plansource, qlist, boundParams, queryEnv); + plan = BuildCachedPlan(plansource, qlist, boundParams, queryEnv, false); /* Accumulate total costs of custom plans */ plansource->total_custom_cost += cached_plan_cost(plan, true); @@ -1387,8 +1393,8 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, } /* - * Reject if AcquireExecutorLocks would have anything to do. This is - * probably unnecessary given the previous check, but let's be safe. + * Reject if there are any lockable relations. This is probably + * unnecessary given the previous check, but let's be safe. */ foreach(lc, plan->stmt_list) { @@ -1776,7 +1782,7 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) foreach(lc1, stmt_list) { PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc1); - ListCell *lc2; + int rtindex; if (plannedstmt->commandType == CMD_UTILITY) { @@ -1794,9 +1800,13 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) continue; } - foreach(lc2, plannedstmt->rtable) + rtindex = -1; + while ((rtindex = bms_next_member(plannedstmt->unprunableRelids, + rtindex)) >= 0) { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc2); + RangeTblEntry *rte = list_nth_node(RangeTblEntry, + plannedstmt->rtable, + rtindex - 1); if (!(rte->rtekind == RTE_RELATION || (rte->rtekind == RTE_SUBQUERY && OidIsValid(rte->relid)))) diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 3ab0aae78f..21c71e0d53 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -103,8 +103,9 @@ extern void ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv); -extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, +extern void ExplainOnePlan(PlannedStmt *plannedstmt, CachedPlan *cplan, + IntoClause *into, ExplainState *es, + const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index dc73de8738..ca39fa1feb 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -45,6 +45,7 @@ extern void ExecCleanupTupleRouting(ModifyTableState *mtstate, * nparts Length of subplan_map[] and subpart_map[]. * subplan_map Subplan index by partition index, or -1. * subpart_map Subpart index by partition index, or -1. + * relid_map Partition OID by partition index. * present_parts A Bitmapset of the partition indexes that we * have subplans or subparts for. * initial_pruning_steps List of PartitionPruneSteps used to @@ -62,6 +63,7 @@ typedef struct PartitionedRelPruningData int nparts; int *subplan_map; int *subpart_map; + Oid *relid_map; Bitmapset *present_parts; List *initial_pruning_steps; List *exec_pruning_steps; @@ -91,6 +93,9 @@ typedef struct PartitionPruningData * the clauses being unable to match to any tuple that the subplan could * possibly produce. * + * lockmode Lock mode to lock the leaf partitions with, if needed; + * this is same as the lock mode that the root partitioned + * table would be locked with. * execparamids Contains paramids of PARAM_EXEC Params found within * any of the partprunedata structs. Pruning must be * done again each time the value of one of these @@ -113,6 +118,7 @@ typedef struct PartitionPruningData */ typedef struct PartitionPruneState { + int lockmode; Bitmapset *execparamids; Bitmapset *other_subplans; MemoryContext prune_context; @@ -128,7 +134,8 @@ extern PartitionPruneState *ExecInitPartitionPruning(PlanState *planstate, Bitmapset *root_parent_relids, Bitmapset **initially_valid_subplans); extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruneState *prunestate, - bool initial_prune); + bool initial_prune, + List **leaf_part_oids); extern PartitionPruneState *CreatePartitionPruneState(EState *estate, PartitionPruneInfo *pruneinfo); #endif /* EXECPARTITION_H */ diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index 0a7274e26c..0e7245435d 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -35,6 +35,7 @@ typedef struct QueryDesc /* These fields are provided by CreateQueryDesc */ CmdType operation; /* CMD_SELECT, CMD_UPDATE, etc. */ PlannedStmt *plannedstmt; /* planner's output (could be utility, too) */ + CachedPlan *cplan; /* CachedPlan that supplies the plannedstmt */ const char *sourceText; /* source text of the query */ Snapshot snapshot; /* snapshot to use for query */ Snapshot crosscheck_snapshot; /* crosscheck for RI update/delete */ @@ -57,6 +58,7 @@ typedef struct QueryDesc /* in pquery.c */ extern QueryDesc *CreateQueryDesc(PlannedStmt *plannedstmt, + CachedPlan *cplan, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index daf04dcf5c..181cf5ad09 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -42,6 +42,7 @@ #include "storage/condition_variable.h" #include "utils/hsearch.h" #include "utils/queryenvironment.h" +#include "utils/plancache.h" #include "utils/reltrigger.h" #include "utils/sharedtuplestore.h" #include "utils/snapshot.h" @@ -635,6 +636,7 @@ typedef struct EState * ExecRowMarks, or NULL if none */ List *es_rteperminfos; /* List of RTEPermissionInfo */ PlannedStmt *es_plannedstmt; /* link to top of plan tree */ + CachedPlan *es_cachedplan; List *es_part_prune_infos; /* PlannedStmt.partPruneInfos */ List *es_part_prune_states; /* List of PartitionPruneState */ List *es_part_prune_results; /* List of Bitmapset */ diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 8d30b6e896..cc2190ea63 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -116,6 +116,12 @@ typedef struct PlannerGlobal /* "flat" rangetable for executor */ List *finalrtable; + /* + * RT indexes of relations subject to removal from the plan due to runtime + * pruning at plan initialization time + */ + Bitmapset *prunableRelids; + /* "flat" list of RTEPermissionInfos */ List *finalrteperminfos; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 39d0281c23..4f552550c8 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -74,6 +74,10 @@ typedef struct PlannedStmt List *rtable; /* list of RangeTblEntry nodes */ + Bitmapset *unprunableRelids; /* RT indexes of relations that are not + * subject to runtime pruning; for + * AcquireExecutorLocks() */ + List *permInfos; /* list of RTEPermissionInfo nodes for rtable * entries needing one */ @@ -1465,6 +1469,13 @@ typedef struct PartitionedRelPruneInfo /* Indexes of all partitions which subplans or subparts are present for */ Bitmapset *present_parts; + /* + * RT indexes of all leaf for which subplans are present; only used during + * planning to help in the construction of PlannerGlobal.prunableRelids + * and set to NULL afterwards to save space in the final plan tree. + */ + Bitmapset *present_leafpart_rtis; + /* Length of the following arrays: */ int nparts; diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h index a90dfdf906..0b5ee007ca 100644 --- a/src/include/utils/plancache.h +++ b/src/include/utils/plancache.h @@ -149,6 +149,7 @@ typedef struct CachedPlan int magic; /* should equal CACHEDPLAN_MAGIC */ List *stmt_list; /* list of PlannedStmts */ bool is_oneshot; /* is it a "oneshot" plan? */ + bool is_generic; /* is it a reusable generic plan? */ bool is_saved; /* is CachedPlan in a long-lived context? */ bool is_valid; /* is the stmt_list currently valid? */ Oid planRoleId; /* Role ID the plan was created for */ @@ -235,4 +236,13 @@ extern bool CachedPlanIsSimplyValid(CachedPlanSource *plansource, extern CachedExpression *GetCachedExpression(Node *expr); extern void FreeCachedExpression(CachedExpression *cexpr); +/* + * CachedPlanRequiresLocking: should the executor acquire locks? + */ +static inline bool +CachedPlanRequiresLocking(CachedPlan *cplan) +{ + return cplan->is_generic; +} + #endif /* PLANCACHE_H */ -- 2.43.0