agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v17 4/8] Row pattern recognition patch (planner). 111+ messages / 2 participants [nested] [flat]
* [PATCH v17 4/8] Row pattern recognition patch (planner). @ 2024-04-28 11:00 Tatsuo Ishii <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Tatsuo Ishii @ 2024-04-28 11:00 UTC (permalink / raw) --- src/backend/optimizer/plan/createplan.c | 24 +++++++++++++++----- src/backend/optimizer/plan/planner.c | 3 +++ src/backend/optimizer/plan/setrefs.c | 27 ++++++++++++++++++++++- src/backend/optimizer/prep/prepjointree.c | 4 ++++ src/include/nodes/plannodes.h | 19 ++++++++++++++++ 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 3b77886567..c8d96be7d7 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -287,9 +287,11 @@ static WindowAgg *make_windowagg(List *tlist, Index winref, int ordNumCols, AttrNumber *ordColIdx, Oid *ordOperators, Oid *ordCollations, int frameOptions, Node *startOffset, Node *endOffset, Oid startInRangeFunc, Oid endInRangeFunc, - Oid inRangeColl, bool inRangeAsc, bool inRangeNullsFirst, - List *runCondition, List *qual, bool topWindow, - Plan *lefttree); + Oid inRangeColl, bool inRangeAsc, bool inRangeNullsFirst, List *runCondition, + RPSkipTo rpSkipTo, List *patternVariable, List *patternRegexp, + List *defineClause, + List *defineInitial, + List *qual, bool topWindow, Plan *lefttree); static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); @@ -2700,6 +2702,11 @@ create_windowagg_plan(PlannerInfo *root, WindowAggPath *best_path) wc->inRangeAsc, wc->inRangeNullsFirst, wc->runCondition, + wc->rpSkipTo, + wc->patternVariable, + wc->patternRegexp, + wc->defineClause, + wc->defineInitial, best_path->qual, best_path->topwindow, subplan); @@ -6629,8 +6636,10 @@ make_windowagg(List *tlist, Index winref, int ordNumCols, AttrNumber *ordColIdx, Oid *ordOperators, Oid *ordCollations, int frameOptions, Node *startOffset, Node *endOffset, Oid startInRangeFunc, Oid endInRangeFunc, - Oid inRangeColl, bool inRangeAsc, bool inRangeNullsFirst, - List *runCondition, List *qual, bool topWindow, Plan *lefttree) + Oid inRangeColl, bool inRangeAsc, bool inRangeNullsFirst, List *runCondition, + RPSkipTo rpSkipTo, List *patternVariable, List *patternRegexp, List *defineClause, + List *defineInitial, + List *qual, bool topWindow, Plan *lefttree) { WindowAgg *node = makeNode(WindowAgg); Plan *plan = &node->plan; @@ -6656,6 +6665,11 @@ make_windowagg(List *tlist, Index winref, node->inRangeAsc = inRangeAsc; node->inRangeNullsFirst = inRangeNullsFirst; node->topWindow = topWindow; + node->rpSkipTo = rpSkipTo, + node->patternVariable = patternVariable; + node->patternRegexp = patternRegexp; + node->defineClause = defineClause; + node->defineInitial = defineInitial; plan->targetlist = tlist; plan->lefttree = lefttree; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 5320da51a0..5eb66c709b 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -873,6 +873,9 @@ subquery_planner(PlannerGlobal *glob, Query *parse, PlannerInfo *parent_root, wc->runCondition = (List *) preprocess_expression(root, (Node *) wc->runCondition, EXPRKIND_TARGET); + wc->defineClause = (List *) preprocess_expression(root, + (Node *) wc->defineClause, + EXPRKIND_TARGET); } parse->limitOffset = preprocess_expression(root, parse->limitOffset, diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 37abcb4701..cd0e7c57d8 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -210,7 +210,6 @@ static List *set_windowagg_runcondition_references(PlannerInfo *root, List *runcondition, Plan *plan); - /***************************************************************************** * * SUBPLAN REFERENCES @@ -2471,6 +2470,32 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) NRM_EQUAL, NUM_EXEC_QUAL(plan)); + /* + * Modifies an expression tree in each DEFINE clause so that all Var + * nodes's varno refers to OUTER_VAR. + */ + if (IsA(plan, WindowAgg)) + { + WindowAgg *wplan = (WindowAgg *) plan; + + if (wplan->defineClause != NIL) + { + foreach(l, wplan->defineClause) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + tle->expr = (Expr *) + fix_upper_expr(root, + (Node *) tle->expr, + subplan_itlist, + OUTER_VAR, + rtoffset, + NRM_EQUAL, + NUM_EXEC_QUAL(plan)); + } + } + } + pfree(subplan_itlist); } diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 41da670f15..b453977178 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -2182,6 +2182,10 @@ perform_pullup_replace_vars(PlannerInfo *root, if (wc->runCondition != NIL) wc->runCondition = (List *) pullup_replace_vars((Node *) wc->runCondition, rvcontext); + + if (wc->defineClause != NIL) + wc->defineClause = (List *) + pullup_replace_vars((Node *) wc->defineClause, rvcontext); } if (parse->onConflict) { diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index e025679f89..90441f4d90 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -20,6 +20,7 @@ #include "lib/stringinfo.h" #include "nodes/bitmapset.h" #include "nodes/lockoptions.h" +#include "nodes/parsenodes.h" #include "nodes/primnodes.h" @@ -1098,6 +1099,24 @@ typedef struct WindowAgg /* nulls sort first for in_range tests? */ bool inRangeNullsFirst; + /* Row Pattern Recognition AFTER MACH SKIP clause */ + RPSkipTo rpSkipTo; /* Row Pattern Skip To type */ + + /* Row Pattern PATTERN variable name (list of String) */ + List *patternVariable; + + /* + * Row Pattern RPATTERN regular expression quantifier ('+' or ''. list of + * String) + */ + List *patternRegexp; + + /* Row Pattern DEFINE clause (list of TargetEntry) */ + List *defineClause; + + /* Row Pattern DEFINE variable initial names (list of String) */ + List *defineInitial; + /* * false for all apart from the WindowAgg that's closest to the root of * the plan -- 2.25.1 ----Next_Part(Sun_Apr_28_20_28_26_2024_444)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0005-Row-pattern-recognition-patch-executor.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 111+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 111+ messages in thread
end of thread, other threads:[~2026-03-31 14:48 UTC | newest] Thread overview: 111+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2024-04-28 11:00 [PATCH v17 4/8] Row pattern recognition patch (planner). Tatsuo Ishii <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox