agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v40 4/4] Use BulkInsertState when copying data to the new heap. 75+ messages / 2 participants [nested] [flat]
* [PATCH v40 4/4] Use BulkInsertState when copying data to the new heap. @ 2026-03-09 10:35 Antonin Houska <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Antonin Houska @ 2026-03-09 10:35 UTC (permalink / raw) It should make the copying more efficient. Besides that, buffer access strategy is involved this way. This diff reverts the changes done in previous parts of the series to reform_and_rewrite_tuple() and introduces a separate function heap_insert_for_repack(). --- src/backend/access/heap/heapam_handler.c | 110 +++++++++++++++-------- 1 file changed, 72 insertions(+), 38 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index f4c62e5bc7a..50ddee9d8c8 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -49,6 +49,11 @@ static void reform_and_rewrite_tuple(HeapTuple tuple, Relation OldHeap, Relation NewHeap, Datum *values, bool *isnull, RewriteState rwstate); +static void heap_insert_for_repack(HeapTuple tuple, Relation OldHeap, + Relation NewHeap, Datum *values, bool *isnull, + BulkInsertState bistate); +static HeapTuple reform_tuple(HeapTuple tuple, Relation OldHeap, + Relation NewHeap, Datum *values, bool *isnull); static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, HeapTuple tuple, @@ -694,6 +699,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, double *tups_recently_dead) { RewriteState rwstate = NULL; + BulkInsertState bistate = NULL; IndexScanDesc indexScan; TableScanDesc tableScan; HeapScanDesc heapScan; @@ -729,6 +735,8 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, if (!concurrent) rwstate = begin_heap_rewrite(OldHeap, NewHeap, OldestXmin, *xid_cutoff, *multi_cutoff); + else + bistate = GetBulkInsertState(); /* Set up sorting if wanted */ @@ -958,8 +966,12 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, }; int64 ct_val[2]; - reform_and_rewrite_tuple(tuple, OldHeap, NewHeap, - values, isnull, rwstate); + if (!concurrent) + reform_and_rewrite_tuple(tuple, OldHeap, NewHeap, + values, isnull, rwstate); + else + heap_insert_for_repack(tuple, OldHeap, NewHeap, + values, isnull, bistate); /* * In indexscan mode and also VACUUM FULL, report increase in @@ -1007,10 +1019,15 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, break; n_tuples += 1; - reform_and_rewrite_tuple(tuple, - OldHeap, NewHeap, - values, isnull, - rwstate); + if (!concurrent) + reform_and_rewrite_tuple(tuple, + OldHeap, NewHeap, + values, isnull, + rwstate); + else + heap_insert_for_repack(tuple, OldHeap, NewHeap, + values, isnull, bistate); + /* Report n_tuples */ pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, n_tuples); @@ -1022,6 +1039,8 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, /* Write out any remaining tuples, and fsync if needed */ if (rwstate) end_heap_rewrite(rwstate); + if (bistate) + FreeBulkInsertState(bistate); /* Clean up */ pfree(values); @@ -2414,56 +2433,71 @@ heapam_scan_sample_next_tuple(TableScanDesc scan, SampleScanState *scanstate, * SET WITHOUT OIDS. * * So, we must reconstruct the tuple from component Datums. - * - * If rwstate=NULL, use simple_heap_insert() instead of rewriting - in that - * case we still need to deform/form the tuple. TODO Shouldn't we rename the - * function, as might not do any rewrite? */ static void reform_and_rewrite_tuple(HeapTuple tuple, Relation OldHeap, Relation NewHeap, Datum *values, bool *isnull, RewriteState rwstate) +{ + HeapTuple copiedTuple; + + copiedTuple = reform_tuple(tuple, OldHeap, NewHeap, values, isnull); + + /* The heap rewrite module does the rest */ + rewrite_heap_tuple(rwstate, tuple, copiedTuple); + + heap_freetuple(copiedTuple); +} + +/* + * Insert tuple when processing REPACK CONCURRENTLY. + * + * rewriteheap.c is not used in the CONCURRENTLY case because it'd be + * difficult to do the same in the catch-up phase (as the logical + * decoding does not provide us with sufficient visibility + * information). Thus we must use heap_insert() both during the + * catch-up and here. + * + * We pass the NO_LOGICAL flag to heap_insert() in order to skip logical + * decoding: as soon as REPACK CONCURRENTLY swaps the relation files, it drops + * this relation, so no logical replication subscription should need the data. + * + * BulkInsertState is used because many tuples are inserted in the typical + * case. + */ +static void +heap_insert_for_repack(HeapTuple tuple, Relation OldHeap, Relation NewHeap, + Datum *values, bool *isnull, BulkInsertState bistate) +{ + tuple = reform_tuple(tuple, OldHeap, NewHeap, values, isnull); + + heap_insert(NewHeap, tuple, GetCurrentCommandId(true), + HEAP_INSERT_NO_LOGICAL, bistate); + + heap_freetuple(tuple); +} + +/* + * Deform tuple, set values of dropped columns to NULL, form a new tuple and + * return it. + */ +static HeapTuple +reform_tuple(HeapTuple tuple, Relation OldHeap, Relation NewHeap, + Datum *values, bool *isnull) { TupleDesc oldTupDesc = RelationGetDescr(OldHeap); TupleDesc newTupDesc = RelationGetDescr(NewHeap); - HeapTuple copiedTuple; int i; heap_deform_tuple(tuple, oldTupDesc, values, isnull); - /* Be sure to null out any dropped columns */ for (i = 0; i < newTupDesc->natts; i++) { if (TupleDescCompactAttr(newTupDesc, i)->attisdropped) isnull[i] = true; } - copiedTuple = heap_form_tuple(newTupDesc, values, isnull); - - if (rwstate) - /* The heap rewrite module does the rest */ - rewrite_heap_tuple(rwstate, tuple, copiedTuple); - else - { - /* - * Insert tuple when processing REPACK CONCURRENTLY. - * - * rewriteheap.c is not used in the CONCURRENTLY case because it'd be - * difficult to do the same in the catch-up phase (as the logical - * decoding does not provide us with sufficient visibility - * information). Thus we must use heap_insert() both during the - * catch-up and here. - * - * The following is like simple_heap_insert() except that we pass the - * flag to skip logical decoding: as soon as REPACK CONCURRENTLY swaps - * the relation files, it drops this relation, so no logical - * replication subscription should need the data. - */ - heap_insert(NewHeap, copiedTuple, GetCurrentCommandId(true), - HEAP_INSERT_NO_LOGICAL, NULL); - } - - heap_freetuple(copiedTuple); + return heap_form_tuple(newTupDesc, values, isnull); } /* -- 2.47.3 --k4rn5ob3nx2ufkij-- ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 75+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 75+ messages in thread
end of thread, other threads:[~2026-03-31 14:48 UTC | newest] Thread overview: 75+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2026-03-09 10:35 [PATCH v40 4/4] Use BulkInsertState when copying data to the new heap. Antonin Houska <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox